drivers/scsi/megaraid/megaraid_sas_fusion.c

   1 /*
   2  *  Linux MegaRAID driver for SAS based RAID controllers
   3  *
   4  *  Copyright (c) 2009-2013  LSI Corporation
   5  *  Copyright (c) 2013-2014  Avago Technologies
   6  *
   7  *  This program is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU General Public License
   9  *  as published by the Free Software Foundation; either version 2
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This program is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *  GNU General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU General Public License
  18  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  *
  20  *  FILE: megaraid_sas_fusion.c
  21  *
  22  *  Authors: Avago Technologies
  23  *           Sumant Patro
  24  *           Adam Radford
  25  *           Kashyap Desai <[email protected]>
  26  *           Sumit Saxena <[email protected]>
  27  *
  28  *  Send feedback to: [email protected]
  29  *
  30  *  Mail to: Avago Technologies, 350 West Trimble Road, Building 90,
  31  *  San Jose, California 95131
  32  */
  33
  34 #include <linux/kernel.h>
  35 #include <linux/types.h>
  36 #include <linux/pci.h>
  37 #include <linux/list.h>
  38 #include <linux/moduleparam.h>
  39 #include <linux/module.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/interrupt.h>
  42 #include <linux/delay.h>
  43 #include <linux/uio.h>
  44 #include <linux/uaccess.h>
  45 #include <linux/fs.h>
  46 #include <linux/compat.h>
  47 #include <linux/blkdev.h>
  48 #include <linux/mutex.h>
  49 #include <linux/poll.h>
  50 #include <linux/vmalloc.h>
  51
  52 #include <scsi/scsi.h>
  53 #include <scsi/scsi_cmnd.h>
  54 #include <scsi/scsi_device.h>
  55 #include <scsi/scsi_host.h>
  56 #include <scsi/scsi_dbg.h>
  57 #include <linux/dmi.h>
  58
  59 #include "megaraid_sas_fusion.h"
  60 #include "megaraid_sas.h"
  61
  62
  63 extern void megasas_free_cmds(struct megasas_instance *instance);
  64 extern struct megasas_cmd *megasas_get_cmd(struct megasas_instance
  65                                            *instance);
  66 extern void
  67 megasas_complete_cmd(struct megasas_instance *instance,
  68                      struct megasas_cmd *cmd, u8 alt_status);
  69 int
  70 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
  71               int seconds);
  72
  73 void
  74 megasas_return_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd);
  75 int megasas_alloc_cmds(struct megasas_instance *instance);
  76 int
  77 megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs);
  78 int
  79 megasas_issue_polled(struct megasas_instance *instance,
  80                      struct megasas_cmd *cmd);
  81 void
  82 megasas_check_and_restore_queue_depth(struct megasas_instance *instance);
  83
  84 int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
  85 void megaraid_sas_kill_hba(struct megasas_instance *instance);
  86
  87 extern u32 megasas_dbg_lvl;
  88 int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
  89                                   int initial);
  90 void megasas_start_timer(struct megasas_instance *instance);
  91 extern struct megasas_mgmt_info megasas_mgmt_info;
  92 extern unsigned int resetwaittime;
  93 extern unsigned int dual_qdepth_disable;
  94 static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
  95 static void megasas_free_reply_fusion(struct megasas_instance *instance);
  96 static inline
  97 void megasas_configure_queue_sizes(struct megasas_instance *instance);
  98
  99 /**
 100  * megasas_check_same_4gb_region -      check if allocation
 101  *                                      crosses same 4GB boundary or not
 102  * @instance -                          adapter's soft instance
 103  * start_addr -                 start address of DMA allocation
 104  * size -                               size of allocation in bytes
 105  * return -                             true : allocation does not cross same
 106  *                                      4GB boundary
 107  *                                      false: allocation crosses same
 108  *                                      4GB boundary
 109  */
 110 static inline bool megasas_check_same_4gb_region
 111         (struct megasas_instance *instance, dma_addr_t start_addr, size_t size)
 112 {
 113         dma_addr_t end_addr;
 114
 115         end_addr = start_addr + size;
 116
 117         if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {
 118                 dev_err(&instance->pdev->dev,
 119                         "Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",
 120                         (unsigned long long)start_addr,
 121                         (unsigned long long)end_addr);
 122                 return false;
 123         }
 124
 125         return true;
 126 }
 127
 128 /**
 129  * megasas_enable_intr_fusion - Enables interrupts
 130  * @regs:                       MFI register set
 131  */
 132 void
 133 megasas_enable_intr_fusion(struct megasas_instance *instance)
 134 {
 135         struct megasas_register_set __iomem *regs;
 136         regs = instance->reg_set;
 137
 138         instance->mask_interrupts = 0;
 139         /* For Thunderbolt/Invader also clear intr on enable */
 140         writel(~0, &regs->outbound_intr_status);
 141         readl(&regs->outbound_intr_status);
 142
 143         writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);
 144
 145         /* Dummy readl to force pci flush */
 146         readl(&regs->outbound_intr_mask);
 147 }
 148
 149 /**
 150  * megasas_disable_intr_fusion - Disables interrupt
 151  * @regs:                        MFI register set
 152  */
 153 void
 154 megasas_disable_intr_fusion(struct megasas_instance *instance)
 155 {
 156         u32 mask = 0xFFFFFFFF;
 157         u32 status;
 158         struct megasas_register_set __iomem *regs;
 159         regs = instance->reg_set;
 160         instance->mask_interrupts = 1;
 161
 162         writel(mask, &regs->outbound_intr_mask);
 163         /* Dummy readl to force pci flush */
 164         status = readl(&regs->outbound_intr_mask);
 165 }
 166
 167 int
 168 megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs)
 169 {
 170         u32 status;
 171         /*
 172          * Check if it is our interrupt
 173          */
 174         status = readl(&regs->outbound_intr_status);
 175
 176         if (status & 1) {
 177                 writel(status, &regs->outbound_intr_status);
 178                 readl(&regs->outbound_intr_status);
 179                 return 1;
 180         }
 181         if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
 182                 return 0;
 183
 184         return 1;
 185 }
 186
 187 /**
 188  * megasas_get_cmd_fusion -     Get a command from the free pool
 189  * @instance:           Adapter soft state
 190  *
 191  * Returns a blk_tag indexed mpt frame
 192  */
 193 inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
 194                                                   *instance, u32 blk_tag)
 195 {
 196         struct fusion_context *fusion;
 197
 198         fusion = instance->ctrl_context;
 199         return fusion->cmd_list[blk_tag];
 200 }
 201
 202 /**
 203  * megasas_return_cmd_fusion -  Return a cmd to free command pool
 204  * @instance:           Adapter soft state
 205  * @cmd:                Command packet to be returned to free command pool
 206  */
 207 inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
 208         struct megasas_cmd_fusion *cmd)
 209 {
 210         cmd->scmd = NULL;
 211         memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
 212         cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
 213         cmd->cmd_completed = false;
 214 }
 215
 216 /**
 217  * megasas_fire_cmd_fusion -    Sends command to the FW
 218  * @instance:                   Adapter soft state
 219  * @req_desc:                   32bit or 64bit Request descriptor
 220  *
 221  * Perform PCI Write. Ventura supports 32 bit Descriptor.
 222  * Prior to Ventura (12G) MR controller supports 64 bit Descriptor.
 223  */
 224
 225 static void
 226 megasas_fire_cmd_fusion(struct megasas_instance *instance,
 227                 union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
 228 {
 229         if (instance->adapter_type == VENTURA_SERIES)
 230                 writel(le32_to_cpu(req_desc->u.low),
 231                         &instance->reg_set->inbound_single_queue_port);
 232         else {
 233 #if defined(writeq) && defined(CONFIG_64BIT)
 234                 u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
 235                                 le32_to_cpu(req_desc->u.low));
 236
 237                 writeq(req_data, &instance->reg_set->inbound_low_queue_port);
 238 #else
 239                 unsigned long flags;
 240                 spin_lock_irqsave(&instance->hba_lock, flags);
 241                 writel(le32_to_cpu(req_desc->u.low),
 242                         &instance->reg_set->inbound_low_queue_port);
 243                 writel(le32_to_cpu(req_desc->u.high),
 244                         &instance->reg_set->inbound_high_queue_port);
 245                 mmiowb();
 246                 spin_unlock_irqrestore(&instance->hba_lock, flags);
 247 #endif
 248         }
 249 }
 250
 251 /**
 252  * megasas_fusion_update_can_queue -    Do all Adapter Queue depth related calculations here
 253  * @instance:                                                   Adapter soft state
 254  * fw_boot_context:                                             Whether this function called during probe or after OCR
 255  *
 256  * This function is only for fusion controllers.
 257  * Update host can queue, if firmware downgrade max supported firmware commands.
 258  * Firmware upgrade case will be skiped because underlying firmware has
 259  * more resource than exposed to the OS.
 260  *
 261  */
 262 static void
 263 megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
 264 {
 265         u16 cur_max_fw_cmds = 0;
 266         u16 ldio_threshold = 0;
 267         struct megasas_register_set __iomem *reg_set;
 268
 269         reg_set = instance->reg_set;
 270
 271         /* ventura FW does not fill outbound_scratch_pad_3 with queue depth */
 272         if (instance->adapter_type < VENTURA_SERIES)
 273                 cur_max_fw_cmds =
 274                 readl(&instance->reg_set->outbound_scratch_pad_3) & 0x00FFFF;
 275
 276         if (dual_qdepth_disable || !cur_max_fw_cmds)
 277                 cur_max_fw_cmds = instance->instancet->read_fw_status_reg(reg_set) & 0x00FFFF;
 278         else
 279                 ldio_threshold =
 280                         (instance->instancet->read_fw_status_reg(reg_set) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
 281
 282         dev_info(&instance->pdev->dev,
 283                  "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",
 284                  cur_max_fw_cmds, ldio_threshold);
 285
 286         if (fw_boot_context == OCR_CONTEXT) {
 287                 cur_max_fw_cmds = cur_max_fw_cmds - 1;
 288                 if (cur_max_fw_cmds < instance->max_fw_cmds) {
 289                         instance->cur_can_queue =
 290                                 cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
 291                                                 MEGASAS_FUSION_IOCTL_CMDS);
 292                         instance->host->can_queue = instance->cur_can_queue;
 293                         instance->ldio_threshold = ldio_threshold;
 294                 }
 295         } else {
 296                 instance->max_fw_cmds = cur_max_fw_cmds;
 297                 instance->ldio_threshold = ldio_threshold;
 298
 299                 if (reset_devices)
 300                         instance->max_fw_cmds = min(instance->max_fw_cmds,
 301                                                 (u16)MEGASAS_KDUMP_QUEUE_DEPTH);
 302                 /*
 303                 * Reduce the max supported cmds by 1. This is to ensure that the
 304                 * reply_q_sz (1 more than the max cmd that driver may send)
 305                 * does not exceed max cmds that the FW can support
 306                 */
 307                 instance->max_fw_cmds = instance->max_fw_cmds-1;
 308         }
 309 }
 310 /**
 311  * megasas_free_cmds_fusion -   Free all the cmds in the free cmd pool
 312  * @instance:           Adapter soft state
 313  */
 314 void
 315 megasas_free_cmds_fusion(struct megasas_instance *instance)
 316 {
 317         int i;
 318         struct fusion_context *fusion = instance->ctrl_context;
 319         struct megasas_cmd_fusion *cmd;
 320
 321         if (fusion->sense)
 322                 dma_pool_free(fusion->sense_dma_pool, fusion->sense,
 323                               fusion->sense_phys_addr);
 324
 325         /* SG */
 326         if (fusion->cmd_list) {
 327                 for (i = 0; i < instance->max_mpt_cmds; i++) {
 328                         cmd = fusion->cmd_list[i];
 329                         if (cmd) {
 330                                 if (cmd->sg_frame)
 331                                         dma_pool_free(fusion->sg_dma_pool,
 332                                                       cmd->sg_frame,
 333                                                       cmd->sg_frame_phys_addr);
 334                         }
 335                         kfree(cmd);
 336                 }
 337                 kfree(fusion->cmd_list);
 338         }
 339
 340         if (fusion->sg_dma_pool) {
 341                 dma_pool_destroy(fusion->sg_dma_pool);
 342                 fusion->sg_dma_pool = NULL;
 343         }
 344         if (fusion->sense_dma_pool) {
 345                 dma_pool_destroy(fusion->sense_dma_pool);
 346                 fusion->sense_dma_pool = NULL;
 347         }
 348
 349
 350         /* Reply Frame, Desc*/
 351         if (instance->is_rdpq)
 352                 megasas_free_rdpq_fusion(instance);
 353         else
 354                 megasas_free_reply_fusion(instance);
 355
 356         /* Request Frame, Desc*/
 357         if (fusion->req_frames_desc)
 358                 dma_free_coherent(&instance->pdev->dev,
 359                         fusion->request_alloc_sz, fusion->req_frames_desc,
 360                         fusion->req_frames_desc_phys);
 361         if (fusion->io_request_frames)
 362                 dma_pool_free(fusion->io_request_frames_pool,
 363                         fusion->io_request_frames,
 364                         fusion->io_request_frames_phys);
 365         if (fusion->io_request_frames_pool) {
 366                 dma_pool_destroy(fusion->io_request_frames_pool);
 367                 fusion->io_request_frames_pool = NULL;
 368         }
 369 }
 370
 371 /**
 372  * megasas_create_sg_sense_fusion -     Creates DMA pool for cmd frames
 373  * @instance:                   Adapter soft state
 374  *
 375  */
 376 static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
 377 {
 378         int i;
 379         u16 max_cmd;
 380         struct fusion_context *fusion;
 381         struct megasas_cmd_fusion *cmd;
 382         int sense_sz;
 383         u32 offset;
 384
 385         fusion = instance->ctrl_context;
 386         max_cmd = instance->max_fw_cmds;
 387         sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;
 388
 389         fusion->sg_dma_pool =
 390                         dma_pool_create("mr_sg", &instance->pdev->dev,
 391                                 instance->max_chain_frame_sz,
 392                                 MR_DEFAULT_NVME_PAGE_SIZE, 0);
 393         /* SCSI_SENSE_BUFFERSIZE  = 96 bytes */
 394         fusion->sense_dma_pool =
 395                         dma_pool_create("mr_sense", &instance->pdev->dev,
 396                                 sense_sz, 64, 0);
 397
 398         if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
 399                 dev_err(&instance->pdev->dev,
 400                         "Failed from %s %d\n",  __func__, __LINE__);
 401                 return -ENOMEM;
 402         }
 403
 404         fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
 405                                        GFP_KERNEL, &fusion->sense_phys_addr);
 406         if (!fusion->sense) {
 407                 dev_err(&instance->pdev->dev,
 408                         "failed from %s %d\n",  __func__, __LINE__);
 409                 return -ENOMEM;
 410         }
 411
 412         /* sense buffer, request frame and reply desc pool requires to be in
 413          * same 4 gb region. Below function will check this.
 414          * In case of failure, new pci pool will be created with updated
 415          * alignment.
 416          * Older allocation and pool will be destroyed.
 417          * Alignment will be used such a way that next allocation if success,
 418          * will always meet same 4gb region requirement.
 419          * Actual requirement is not alignment, but we need start and end of
 420          * DMA address must have same upper 32 bit address.
 421          */
 422
 423         if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr,
 424                                            sense_sz)) {
 425                 dma_pool_free(fusion->sense_dma_pool, fusion->sense,
 426                               fusion->sense_phys_addr);
 427                 fusion->sense = NULL;
 428                 dma_pool_destroy(fusion->sense_dma_pool);
 429
 430                 fusion->sense_dma_pool =
 431                         dma_pool_create("mr_sense_align", &instance->pdev->dev,
 432                                         sense_sz, roundup_pow_of_two(sense_sz),
 433                                         0);
 434                 if (!fusion->sense_dma_pool) {
 435                         dev_err(&instance->pdev->dev,
 436                                 "Failed from %s %d\n",  __func__, __LINE__);
 437                         return -ENOMEM;
 438                 }
 439                 fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
 440                                                GFP_KERNEL,
 441                                                &fusion->sense_phys_addr);
 442                 if (!fusion->sense) {
 443                         dev_err(&instance->pdev->dev,
 444                                 "failed from %s %d\n",  __func__, __LINE__);
 445                         return -ENOMEM;
 446                 }
 447         }
 448
 449         /*
 450          * Allocate and attach a frame to each of the commands in cmd_list
 451          */
 452         for (i = 0; i < max_cmd; i++) {
 453                 cmd = fusion->cmd_list[i];
 454                 cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,
 455                                         GFP_KERNEL, &cmd->sg_frame_phys_addr);
 456
 457                 offset = SCSI_SENSE_BUFFERSIZE * i;
 458                 cmd->sense = (u8 *)fusion->sense + offset;
 459                 cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
 460
 461                 if (!cmd->sg_frame) {
 462                         dev_err(&instance->pdev->dev,
 463                                 "Failed from %s %d\n",  __func__, __LINE__);
 464                         return -ENOMEM;
 465                 }
 466         }
 467
 468         /* create sense buffer for the raid 1/10 fp */
 469         for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
 470                 cmd = fusion->cmd_list[i];
 471                 offset = SCSI_SENSE_BUFFERSIZE * i;
 472                 cmd->sense = (u8 *)fusion->sense + offset;
 473                 cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
 474
 475         }
 476
 477         return 0;
 478 }
 479
 480 int
 481 megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
 482 {
 483         u32 max_mpt_cmd, i, j;
 484         struct fusion_context *fusion;
 485
 486         fusion = instance->ctrl_context;
 487
 488         max_mpt_cmd = instance->max_mpt_cmds;
 489
 490         /*
 491          * fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.
 492          * Allocate the dynamic array first and then allocate individual
 493          * commands.
 494          */
 495         fusion->cmd_list =
 496                 kzalloc(sizeof(struct megasas_cmd_fusion *) * max_mpt_cmd,
 497                         GFP_KERNEL);
 498         if (!fusion->cmd_list) {
 499                 dev_err(&instance->pdev->dev,
 500                         "Failed from %s %d\n",  __func__, __LINE__);
 501                 return -ENOMEM;
 502         }
 503
 504         for (i = 0; i < max_mpt_cmd; i++) {
 505                 fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),
 506                                               GFP_KERNEL);
 507                 if (!fusion->cmd_list[i]) {
 508                         for (j = 0; j < i; j++)
 509                                 kfree(fusion->cmd_list[j]);
 510                         kfree(fusion->cmd_list);
 511                         dev_err(&instance->pdev->dev,
 512                                 "Failed from %s %d\n",  __func__, __LINE__);
 513                         return -ENOMEM;
 514                 }
 515         }
 516
 517         return 0;
 518 }
 519 int
 520 megasas_alloc_request_fusion(struct megasas_instance *instance)
 521 {
 522         struct fusion_context *fusion;
 523
 524         fusion = instance->ctrl_context;
 525
 526 retry_alloc:
 527         fusion->io_request_frames_pool =
 528                         dma_pool_create("mr_ioreq", &instance->pdev->dev,
 529                                 fusion->io_frames_alloc_sz, 16, 0);
 530
 531         if (!fusion->io_request_frames_pool) {
 532                 dev_err(&instance->pdev->dev,
 533                         "Failed from %s %d\n",  __func__, __LINE__);
 534                 return -ENOMEM;
 535         }
 536
 537         fusion->io_request_frames =
 538                         dma_pool_alloc(fusion->io_request_frames_pool,
 539                                 GFP_KERNEL, &fusion->io_request_frames_phys);
 540         if (!fusion->io_request_frames) {
 541                 if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {
 542                         instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;
 543                         dma_pool_destroy(fusion->io_request_frames_pool);
 544                         megasas_configure_queue_sizes(instance);
 545                         goto retry_alloc;
 546                 } else {
 547                         dev_err(&instance->pdev->dev,
 548                                 "Failed from %s %d\n",  __func__, __LINE__);
 549                         return -ENOMEM;
 550                 }
 551         }
 552
 553         if (!megasas_check_same_4gb_region(instance,
 554                                            fusion->io_request_frames_phys,
 555                                            fusion->io_frames_alloc_sz)) {
 556                 dma_pool_free(fusion->io_request_frames_pool,
 557                               fusion->io_request_frames,
 558                               fusion->io_request_frames_phys);
 559                 fusion->io_request_frames = NULL;
 560                 dma_pool_destroy(fusion->io_request_frames_pool);
 561
 562                 fusion->io_request_frames_pool =
 563                         dma_pool_create("mr_ioreq_align",
 564                                         &instance->pdev->dev,
 565                                         fusion->io_frames_alloc_sz,
 566                                         roundup_pow_of_two(fusion->io_frames_alloc_sz),
 567                                         0);
 568
 569                 if (!fusion->io_request_frames_pool) {
 570                         dev_err(&instance->pdev->dev,
 571                                 "Failed from %s %d\n",  __func__, __LINE__);
 572                         return -ENOMEM;
 573                 }
 574
 575                 fusion->io_request_frames =
 576                         dma_pool_alloc(fusion->io_request_frames_pool,
 577                                        GFP_KERNEL,
 578                                        &fusion->io_request_frames_phys);
 579
 580                 if (!fusion->io_request_frames) {
 581                         dev_err(&instance->pdev->dev,
 582                                 "Failed from %s %d\n",  __func__, __LINE__);
 583                         return -ENOMEM;
 584                 }
 585         }
 586
 587         fusion->req_frames_desc =
 588                 dma_alloc_coherent(&instance->pdev->dev,
 589                                    fusion->request_alloc_sz,
 590                                    &fusion->req_frames_desc_phys, GFP_KERNEL);
 591         if (!fusion->req_frames_desc) {
 592                 dev_err(&instance->pdev->dev,
 593                         "Failed from %s %d\n",  __func__, __LINE__);
 594                 return -ENOMEM;
 595         }
 596
 597         return 0;
 598 }
 599
 600 int
 601 megasas_alloc_reply_fusion(struct megasas_instance *instance)
 602 {
 603         int i, count;
 604         struct fusion_context *fusion;
 605         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
 606         fusion = instance->ctrl_context;
 607
 608         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 609         fusion->reply_frames_desc_pool =
 610                         dma_pool_create("mr_reply", &instance->pdev->dev,
 611                                 fusion->reply_alloc_sz * count, 16, 0);
 612
 613         if (!fusion->reply_frames_desc_pool) {
 614                 dev_err(&instance->pdev->dev,
 615                         "Failed from %s %d\n",  __func__, __LINE__);
 616                 return -ENOMEM;
 617         }
 618
 619         fusion->reply_frames_desc[0] =
 620                 dma_pool_alloc(fusion->reply_frames_desc_pool,
 621                         GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
 622         if (!fusion->reply_frames_desc[0]) {
 623                 dev_err(&instance->pdev->dev,
 624                         "Failed from %s %d\n",  __func__, __LINE__);
 625                 return -ENOMEM;
 626         }
 627
 628         if (!megasas_check_same_4gb_region(instance,
 629                                            fusion->reply_frames_desc_phys[0],
 630                                            (fusion->reply_alloc_sz * count))) {
 631                 dma_pool_free(fusion->reply_frames_desc_pool,
 632                               fusion->reply_frames_desc[0],
 633                               fusion->reply_frames_desc_phys[0]);
 634                 fusion->reply_frames_desc[0] = NULL;
 635                 dma_pool_destroy(fusion->reply_frames_desc_pool);
 636
 637                 fusion->reply_frames_desc_pool =
 638                         dma_pool_create("mr_reply_align",
 639                                         &instance->pdev->dev,
 640                                         fusion->reply_alloc_sz * count,
 641                                         roundup_pow_of_two(fusion->reply_alloc_sz * count),
 642                                         0);
 643
 644                 if (!fusion->reply_frames_desc_pool) {
 645                         dev_err(&instance->pdev->dev,
 646                                 "Failed from %s %d\n",  __func__, __LINE__);
 647                         return -ENOMEM;
 648                 }
 649
 650                 fusion->reply_frames_desc[0] =
 651                         dma_pool_alloc(fusion->reply_frames_desc_pool,
 652                                        GFP_KERNEL,
 653                                        &fusion->reply_frames_desc_phys[0]);
 654
 655                 if (!fusion->reply_frames_desc[0]) {
 656                         dev_err(&instance->pdev->dev,
 657                                 "Failed from %s %d\n",  __func__, __LINE__);
 658                         return -ENOMEM;
 659                 }
 660         }
 661
 662         reply_desc = fusion->reply_frames_desc[0];
 663         for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
 664                 reply_desc->Words = cpu_to_le64(ULLONG_MAX);
 665
 666         /* This is not a rdpq mode, but driver still populate
 667          * reply_frame_desc array to use same msix index in ISR path.
 668          */
 669         for (i = 0; i < (count - 1); i++)
 670                 fusion->reply_frames_desc[i + 1] =
 671                         fusion->reply_frames_desc[i] +
 672                         (fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
 673
 674         return 0;
 675 }
 676
 677 int
 678 megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
 679 {
 680         int i, j, k, msix_count;
 681         struct fusion_context *fusion;
 682         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
 683         union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];
 684         dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];
 685         u8 dma_alloc_count, abs_index;
 686         u32 chunk_size, array_size, offset;
 687
 688         fusion = instance->ctrl_context;
 689         chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
 690         array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
 691                      MAX_MSIX_QUEUES_FUSION;
 692
 693         fusion->rdpq_virt = pci_alloc_consistent(instance->pdev, array_size,
 694                                                  &fusion->rdpq_phys);
 695         if (!fusion->rdpq_virt) {
 696                 dev_err(&instance->pdev->dev,
 697                         "Failed from %s %d\n",  __func__, __LINE__);
 698                 return -ENOMEM;
 699         }
 700
 701         memset(fusion->rdpq_virt, 0, array_size);
 702         msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 703
 704         fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
 705                                                          &instance->pdev->dev,
 706                                                          chunk_size, 16, 0);
 707         fusion->reply_frames_desc_pool_align =
 708                                 dma_pool_create("mr_rdpq_align",
 709                                                 &instance->pdev->dev,
 710                                                 chunk_size,
 711                                                 roundup_pow_of_two(chunk_size),
 712                                                 0);
 713
 714         if (!fusion->reply_frames_desc_pool ||
 715             !fusion->reply_frames_desc_pool_align) {
 716                 dev_err(&instance->pdev->dev,
 717                         "Failed from %s %d\n",  __func__, __LINE__);
 718                 return -ENOMEM;
 719         }
 720
 721 /*
 722  * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
 723  * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be
 724  * within 4GB boundary and also reply queues in a set must have same
 725  * upper 32-bits in their memory address. so here driver is allocating the
 726  * DMA'able memory for reply queues according. Driver uses limitation of
 727  * VENTURA_SERIES to manage INVADER_SERIES as well.
 728  */
 729         dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);
 730
 731         for (i = 0; i < dma_alloc_count; i++) {
 732                 rdpq_chunk_virt[i] =
 733                         dma_pool_alloc(fusion->reply_frames_desc_pool,
 734                                        GFP_KERNEL, &rdpq_chunk_phys[i]);
 735                 if (!rdpq_chunk_virt[i]) {
 736                         dev_err(&instance->pdev->dev,
 737                                 "Failed from %s %d\n",  __func__, __LINE__);
 738                         return -ENOMEM;
 739                 }
 740                 /* reply desc pool requires to be in same 4 gb region.
 741                  * Below function will check this.
 742                  * In case of failure, new pci pool will be created with updated
 743                  * alignment.
 744                  * For RDPQ buffers, driver always allocate two separate pci pool.
 745                  * Alignment will be used such a way that next allocation if
 746                  * success, will always meet same 4gb region requirement.
 747                  * rdpq_tracker keep track of each buffer's physical,
 748                  * virtual address and pci pool descriptor. It will help driver
 749                  * while freeing the resources.
 750                  *
 751                  */
 752                 if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i],
 753                                                    chunk_size)) {
 754                         dma_pool_free(fusion->reply_frames_desc_pool,
 755                                       rdpq_chunk_virt[i],
 756                                       rdpq_chunk_phys[i]);
 757
 758                         rdpq_chunk_virt[i] =
 759                                 dma_pool_alloc(fusion->reply_frames_desc_pool_align,
 760                                                GFP_KERNEL, &rdpq_chunk_phys[i]);
 761                         if (!rdpq_chunk_virt[i]) {
 762                                 dev_err(&instance->pdev->dev,
 763                                         "Failed from %s %d\n",
 764                                         __func__, __LINE__);
 765                                 return -ENOMEM;
 766                         }
 767                         fusion->rdpq_tracker[i].dma_pool_ptr =
 768                                         fusion->reply_frames_desc_pool_align;
 769                 } else {
 770                         fusion->rdpq_tracker[i].dma_pool_ptr =
 771                                         fusion->reply_frames_desc_pool;
 772                 }
 773
 774                 fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];
 775                 fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];
 776         }
 777
 778         for (k = 0; k < dma_alloc_count; k++) {
 779                 for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {
 780                         abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;
 781
 782                         if (abs_index == msix_count)
 783                                 break;
 784                         offset = fusion->reply_alloc_sz * i;
 785                         fusion->rdpq_virt[abs_index].RDPQBaseAddress =
 786                                         cpu_to_le64(rdpq_chunk_phys[k] + offset);
 787                         fusion->reply_frames_desc_phys[abs_index] =
 788                                         rdpq_chunk_phys[k] + offset;
 789                         fusion->reply_frames_desc[abs_index] =
 790                                         (union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);
 791
 792                         reply_desc = fusion->reply_frames_desc[abs_index];
 793                         for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
 794                                 reply_desc->Words = ULLONG_MAX;
 795                 }
 796         }
 797
 798         return 0;
 799 }
 800
 801 static void
 802 megasas_free_rdpq_fusion(struct megasas_instance *instance) {
 803
 804         int i;
 805         struct fusion_context *fusion;
 806
 807         fusion = instance->ctrl_context;
 808
 809         for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {
 810                 if (fusion->rdpq_tracker[i].pool_entry_virt)
 811                         dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr,
 812                                       fusion->rdpq_tracker[i].pool_entry_virt,
 813                                       fusion->rdpq_tracker[i].pool_entry_phys);
 814
 815         }
 816
 817         if (fusion->reply_frames_desc_pool)
 818                 dma_pool_destroy(fusion->reply_frames_desc_pool);
 819         if (fusion->reply_frames_desc_pool_align)
 820                 dma_pool_destroy(fusion->reply_frames_desc_pool_align);
 821
 822         if (fusion->rdpq_virt)
 823                 pci_free_consistent(instance->pdev,
 824                         sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
 825                         fusion->rdpq_virt, fusion->rdpq_phys);
 826 }
 827
 828 static void
 829 megasas_free_reply_fusion(struct megasas_instance *instance) {
 830
 831         struct fusion_context *fusion;
 832
 833         fusion = instance->ctrl_context;
 834
 835         if (fusion->reply_frames_desc[0])
 836                 dma_pool_free(fusion->reply_frames_desc_pool,
 837                         fusion->reply_frames_desc[0],
 838                         fusion->reply_frames_desc_phys[0]);
 839
 840         if (fusion->reply_frames_desc_pool)
 841                 dma_pool_destroy(fusion->reply_frames_desc_pool);
 842
 843 }
 844
 845
 846 /**
 847  * megasas_alloc_cmds_fusion -  Allocates the command packets
 848  * @instance:           Adapter soft state
 849  *
 850  *
 851  * Each frame has a 32-bit field called context. This context is used to get
 852  * back the megasas_cmd_fusion from the frame when a frame gets completed
 853  * In this driver, the 32 bit values are the indices into an array cmd_list.
 854  * This array is used only to look up the megasas_cmd_fusion given the context.
 855  * The free commands themselves are maintained in a linked list called cmd_pool.
 856  *
 857  * cmds are formed in the io_request and sg_frame members of the
 858  * megasas_cmd_fusion. The context field is used to get a request descriptor
 859  * and is used as SMID of the cmd.
 860  * SMID value range is from 1 to max_fw_cmds.
 861  */
 862 int
 863 megasas_alloc_cmds_fusion(struct megasas_instance *instance)
 864 {
 865         int i;
 866         struct fusion_context *fusion;
 867         struct megasas_cmd_fusion *cmd;
 868         u32 offset;
 869         dma_addr_t io_req_base_phys;
 870         u8 *io_req_base;
 871
 872
 873         fusion = instance->ctrl_context;
 874
 875         if (megasas_alloc_request_fusion(instance))
 876                 goto fail_exit;
 877
 878         if (instance->is_rdpq) {
 879                 if (megasas_alloc_rdpq_fusion(instance))
 880                         goto fail_exit;
 881         } else
 882                 if (megasas_alloc_reply_fusion(instance))
 883                         goto fail_exit;
 884
 885         if (megasas_alloc_cmdlist_fusion(instance))
 886                 goto fail_exit;
 887
 888         dev_info(&instance->pdev->dev, "Configured max firmware commands: %d\n",
 889                  instance->max_fw_cmds);
 890
 891         /* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */
 892         io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
 893         io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
 894
 895         /*
 896          * Add all the commands to command pool (fusion->cmd_pool)
 897          */
 898
 899         /* SMID 0 is reserved. Set SMID/index from 1 */
 900         for (i = 0; i < instance->max_mpt_cmds; i++) {
 901                 cmd = fusion->cmd_list[i];
 902                 offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
 903                 memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
 904                 cmd->index = i + 1;
 905                 cmd->scmd = NULL;
 906                 cmd->sync_cmd_idx =
 907                 (i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
 908                                 (i - instance->max_scsi_cmds) :
 909                                 (u32)ULONG_MAX; /* Set to Invalid */
 910                 cmd->instance = instance;
 911                 cmd->io_request =
 912                         (struct MPI2_RAID_SCSI_IO_REQUEST *)
 913                   (io_req_base + offset);
 914                 memset(cmd->io_request, 0,
 915                        sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
 916                 cmd->io_request_phys_addr = io_req_base_phys + offset;
 917                 cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
 918         }
 919
 920         if (megasas_create_sg_sense_fusion(instance))
 921                 goto fail_exit;
 922
 923         return 0;
 924
 925 fail_exit:
 926         megasas_free_cmds_fusion(instance);
 927         return -ENOMEM;
 928 }
 929
 930 /**
 931  * wait_and_poll -      Issues a polling command
 932  * @instance:                   Adapter soft state
 933  * @cmd:                        Command packet to be issued
 934  *
 935  * For polling, MFI requires the cmd_status to be set to 0xFF before posting.
 936  */
 937 int
 938 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
 939         int seconds)
 940 {
 941         int i;
 942         struct megasas_header *frame_hdr = &cmd->frame->hdr;
 943         struct fusion_context *fusion;
 944
 945         u32 msecs = seconds * 1000;
 946
 947         fusion = instance->ctrl_context;
 948         /*
 949          * Wait for cmd_status to change
 950          */
 951         for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
 952                 rmb();
 953                 msleep(20);
 954         }
 955
 956         if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
 957                 return DCMD_TIMEOUT;
 958         else if (frame_hdr->cmd_status == MFI_STAT_OK)
 959                 return DCMD_SUCCESS;
 960         else
 961                 return DCMD_FAILED;
 962 }
 963
 964 /**
 965  * megasas_ioc_init_fusion -    Initializes the FW
 966  * @instance:           Adapter soft state
 967  *
 968  * Issues the IOC Init cmd
 969  */
 970 int
 971 megasas_ioc_init_fusion(struct megasas_instance *instance)
 972 {
 973         struct megasas_init_frame *init_frame;
 974         struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
 975         dma_addr_t      ioc_init_handle;
 976         struct megasas_cmd *cmd;
 977         u8 ret, cur_rdpq_mode;
 978         struct fusion_context *fusion;
 979         union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
 980         int i;
 981         struct megasas_header *frame_hdr;
 982         const char *sys_info;
 983         MFI_CAPABILITIES *drv_ops;
 984         u32 scratch_pad_2;
 985         unsigned long flags;
 986         ktime_t time;
 987         bool cur_fw_64bit_dma_capable;
 988
 989         fusion = instance->ctrl_context;
 990
 991         ioc_init_handle = fusion->ioc_init_request_phys;
 992         IOCInitMessage = fusion->ioc_init_request;
 993
 994         cmd = fusion->ioc_init_cmd;
 995
 996         scratch_pad_2 = readl
 997                 (&instance->reg_set->outbound_scratch_pad_2);
 998
 999         cur_rdpq_mode = (scratch_pad_2 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
1000
1001         if (instance->adapter_type == INVADER_SERIES) {
1002                 cur_fw_64bit_dma_capable =
1003                         (scratch_pad_2 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;
1004
1005                 if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {
1006                         dev_err(&instance->pdev->dev, "Driver was operating on 64bit "
1007                                 "DMA mask, but upcoming FW does not support 64bit DMA mask\n");
1008                         megaraid_sas_kill_hba(instance);
1009                         ret = 1;
1010                         goto fail_fw_init;
1011                 }
1012         }
1013
1014         if (instance->is_rdpq && !cur_rdpq_mode) {
1015                 dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
1016                         " from RDPQ mode to non RDPQ mode\n");
1017                 ret = 1;
1018                 goto fail_fw_init;
1019         }
1020
1021         instance->fw_sync_cache_support = (scratch_pad_2 &
1022                 MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
1023         dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
1024                  instance->fw_sync_cache_support ? "Yes" : "No");
1025
1026         memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
1027
1028         IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
1029         IOCInitMessage->WhoInit = MPI2_WHOINIT_HOST_DRIVER;
1030         IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
1031         IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
1032         IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
1033
1034         IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
1035         IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
1036                         cpu_to_le64(fusion->rdpq_phys) :
1037                         cpu_to_le64(fusion->reply_frames_desc_phys[0]);
1038         IOCInitMessage->MsgFlags = instance->is_rdpq ?
1039                         MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
1040         IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
1041         IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));
1042         IOCInitMessage->HostMSIxVectors = instance->msix_vectors;
1043         IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
1044
1045         time = ktime_get_real();
1046         /* Convert to milliseconds as per FW requirement */
1047         IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));
1048
1049         init_frame = (struct megasas_init_frame *)cmd->frame;
1050         memset(init_frame, 0, IOC_INIT_FRAME_SIZE);
1051
1052         frame_hdr = &cmd->frame->hdr;
1053         frame_hdr->cmd_status = 0xFF;
1054         frame_hdr->flags = cpu_to_le16(
1055                 le16_to_cpu(frame_hdr->flags) |
1056                 MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
1057
1058         init_frame->cmd = MFI_CMD_INIT;
1059         init_frame->cmd_status = 0xFF;
1060
1061         drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
1062
1063         /* driver support Extended MSIX */
1064         if (instance->adapter_type >= INVADER_SERIES)
1065                 drv_ops->mfi_capabilities.support_additional_msix = 1;
1066         /* driver supports HA / Remote LUN over Fast Path interface */
1067         drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
1068
1069         drv_ops->mfi_capabilities.support_max_255lds = 1;
1070         drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
1071         drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
1072
1073         if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
1074                 drv_ops->mfi_capabilities.support_ext_io_size = 1;
1075
1076         drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
1077         if (!dual_qdepth_disable)
1078                 drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
1079
1080         drv_ops->mfi_capabilities.support_qd_throttling = 1;
1081         drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
1082         drv_ops->mfi_capabilities.support_nvme_passthru = 1;
1083
1084         if (instance->consistent_mask_64bit)
1085                 drv_ops->mfi_capabilities.support_64bit_mode = 1;
1086
1087         /* Convert capability to LE32 */
1088         cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
1089
1090         sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);
1091         if (instance->system_info_buf && sys_info) {
1092                 memcpy(instance->system_info_buf->systemId, sys_info,
1093                         strlen(sys_info) > 64 ? 64 : strlen(sys_info));
1094                 instance->system_info_buf->systemIdLength =
1095                         strlen(sys_info) > 64 ? 64 : strlen(sys_info);
1096                 init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));
1097                 init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));
1098         }
1099
1100         init_frame->queue_info_new_phys_addr_hi =
1101                 cpu_to_le32(upper_32_bits(ioc_init_handle));
1102         init_frame->queue_info_new_phys_addr_lo =
1103                 cpu_to_le32(lower_32_bits(ioc_init_handle));
1104         init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
1105
1106         req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
1107         req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
1108         req_desc.MFAIo.RequestFlags =
1109                 (MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
1110                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1111
1112         /*
1113          * disable the intr before firing the init frame
1114          */
1115         instance->instancet->disable_intr(instance);
1116
1117         for (i = 0; i < (10 * 1000); i += 20) {
1118                 if (readl(&instance->reg_set->doorbell) & 1)
1119                         msleep(20);
1120                 else
1121                         break;
1122         }
1123
1124         /* For Ventura also IOC INIT required 64 bit Descriptor write. */
1125         spin_lock_irqsave(&instance->hba_lock, flags);
1126         writel(le32_to_cpu(req_desc.u.low),
1127                &instance->reg_set->inbound_low_queue_port);
1128         writel(le32_to_cpu(req_desc.u.high),
1129                &instance->reg_set->inbound_high_queue_port);
1130         mmiowb();
1131         spin_unlock_irqrestore(&instance->hba_lock, flags);
1132
1133         wait_and_poll(instance, cmd, MFI_POLL_TIMEOUT_SECS);
1134
1135         frame_hdr = &cmd->frame->hdr;
1136         if (frame_hdr->cmd_status != 0) {
1137                 ret = 1;
1138                 goto fail_fw_init;
1139         }
1140
1141         ret = 0;
1142
1143 fail_fw_init:
1144         dev_err(&instance->pdev->dev,
1145                 "Init cmd return status %s for SCSI host %d\n",
1146                 ret ? "FAILED" : "SUCCESS", instance->host->host_no);
1147
1148         return ret;
1149 }
1150
1151 /**
1152  * megasas_sync_pd_seq_num -    JBOD SEQ MAP
1153  * @instance:           Adapter soft state
1154  * @pend:               set to 1, if it is pended jbod map.
1155  *
1156  * Issue Jbod map to the firmware. If it is pended command,
1157  * issue command and return. If it is first instance of jbod map
1158  * issue and receive command.
1159  */
1160 int
1161 megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
1162         int ret = 0;
1163         u32 pd_seq_map_sz;
1164         struct megasas_cmd *cmd;
1165         struct megasas_dcmd_frame *dcmd;
1166         struct fusion_context *fusion = instance->ctrl_context;
1167         struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1168         dma_addr_t pd_seq_h;
1169
1170         pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
1171         pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
1172         pd_seq_map_sz = sizeof(struct MR_PD_CFG_SEQ_NUM_SYNC) +
1173                         (sizeof(struct MR_PD_CFG_SEQ) *
1174                         (MAX_PHYSICAL_DEVICES - 1));
1175
1176         cmd = megasas_get_cmd(instance);
1177         if (!cmd) {
1178                 dev_err(&instance->pdev->dev,
1179                         "Could not get mfi cmd. Fail from %s %d\n",
1180                         __func__, __LINE__);
1181                 return -ENOMEM;
1182         }
1183
1184         dcmd = &cmd->frame->dcmd;
1185
1186         memset(pd_sync, 0, pd_seq_map_sz);
1187         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1188
1189         if (pend) {
1190                 dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1191                 dcmd->flags = MFI_FRAME_DIR_WRITE;
1192                 instance->jbod_seq_cmd = cmd;
1193         } else {
1194                 dcmd->flags = MFI_FRAME_DIR_READ;
1195         }
1196
1197         dcmd->cmd = MFI_CMD_DCMD;
1198         dcmd->cmd_status = 0xFF;
1199         dcmd->sge_count = 1;
1200         dcmd->timeout = 0;
1201         dcmd->pad_0 = 0;
1202         dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
1203         dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
1204
1205         megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz);
1206
1207         if (pend) {
1208                 instance->instancet->issue_dcmd(instance, cmd);
1209                 return 0;
1210         }
1211
1212         /* Below code is only for non pended DCMD */
1213         if (!instance->mask_interrupts)
1214                 ret = megasas_issue_blocked_cmd(instance, cmd,
1215                         MFI_IO_TIMEOUT_SECS);
1216         else
1217                 ret = megasas_issue_polled(instance, cmd);
1218
1219         if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
1220                 dev_warn(&instance->pdev->dev,
1221                         "driver supports max %d JBOD, but FW reports %d\n",
1222                         MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
1223                 ret = -EINVAL;
1224         }
1225
1226         if (ret == DCMD_TIMEOUT)
1227                 megaraid_sas_kill_hba(instance);
1228
1229         if (ret == DCMD_SUCCESS)
1230                 instance->pd_seq_map_id++;
1231
1232         megasas_return_cmd(instance, cmd);
1233         return ret;
1234 }
1235
1236 /*
1237  * megasas_get_ld_map_info -    Returns FW's ld_map structure
1238  * @instance:                           Adapter soft state
1239  * @pend:                               Pend the command or not
1240  * Issues an internal command (DCMD) to get the FW's controller PD
1241  * list structure.  This information is mainly used to find out SYSTEM
1242  * supported by the FW.
1243  * dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO
1244  * dcmd.mbox.b[0]       - number of LDs being sync'd
1245  * dcmd.mbox.b[1]       - 0 - complete command immediately.
1246  *                      - 1 - pend till config change
1247  * dcmd.mbox.b[2]       - 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP
1248  *                      - 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and
1249  *                              uses extended struct MR_FW_RAID_MAP_EXT
1250  */
1251 static int
1252 megasas_get_ld_map_info(struct megasas_instance *instance)
1253 {
1254         int ret = 0;
1255         struct megasas_cmd *cmd;
1256         struct megasas_dcmd_frame *dcmd;
1257         void *ci;
1258         dma_addr_t ci_h = 0;
1259         u32 size_map_info;
1260         struct fusion_context *fusion;
1261
1262         cmd = megasas_get_cmd(instance);
1263
1264         if (!cmd) {
1265                 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1266                 return -ENOMEM;
1267         }
1268
1269         fusion = instance->ctrl_context;
1270
1271         if (!fusion) {
1272                 megasas_return_cmd(instance, cmd);
1273                 return -ENXIO;
1274         }
1275
1276         dcmd = &cmd->frame->dcmd;
1277
1278         size_map_info = fusion->current_map_sz;
1279
1280         ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1281         ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1282
1283         if (!ci) {
1284                 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1285                 megasas_return_cmd(instance, cmd);
1286                 return -ENOMEM;
1287         }
1288
1289         memset(ci, 0, fusion->max_map_sz);
1290         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1291         dcmd->cmd = MFI_CMD_DCMD;
1292         dcmd->cmd_status = 0xFF;
1293         dcmd->sge_count = 1;
1294         dcmd->flags = MFI_FRAME_DIR_READ;
1295         dcmd->timeout = 0;
1296         dcmd->pad_0 = 0;
1297         dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1298         dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1299
1300         megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1301
1302         if (!instance->mask_interrupts)
1303                 ret = megasas_issue_blocked_cmd(instance, cmd,
1304                         MFI_IO_TIMEOUT_SECS);
1305         else
1306                 ret = megasas_issue_polled(instance, cmd);
1307
1308         if (ret == DCMD_TIMEOUT)
1309                 megaraid_sas_kill_hba(instance);
1310
1311         megasas_return_cmd(instance, cmd);
1312
1313         return ret;
1314 }
1315
1316 u8
1317 megasas_get_map_info(struct megasas_instance *instance)
1318 {
1319         struct fusion_context *fusion = instance->ctrl_context;
1320
1321         fusion->fast_path_io = 0;
1322         if (!megasas_get_ld_map_info(instance)) {
1323                 if (MR_ValidateMapInfo(instance, instance->map_id)) {
1324                         fusion->fast_path_io = 1;
1325                         return 0;
1326                 }
1327         }
1328         return 1;
1329 }
1330
1331 /*
1332  * megasas_sync_map_info -      Returns FW's ld_map structure
1333  * @instance:                           Adapter soft state
1334  *
1335  * Issues an internal command (DCMD) to get the FW's controller PD
1336  * list structure.  This information is mainly used to find out SYSTEM
1337  * supported by the FW.
1338  */
1339 int
1340 megasas_sync_map_info(struct megasas_instance *instance)
1341 {
1342         int i;
1343         struct megasas_cmd *cmd;
1344         struct megasas_dcmd_frame *dcmd;
1345         u16 num_lds;
1346         u32 size_sync_info;
1347         struct fusion_context *fusion;
1348         struct MR_LD_TARGET_SYNC *ci = NULL;
1349         struct MR_DRV_RAID_MAP_ALL *map;
1350         struct MR_LD_RAID  *raid;
1351         struct MR_LD_TARGET_SYNC *ld_sync;
1352         dma_addr_t ci_h = 0;
1353         u32 size_map_info;
1354
1355         cmd = megasas_get_cmd(instance);
1356
1357         if (!cmd) {
1358                 dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1359                 return -ENOMEM;
1360         }
1361
1362         fusion = instance->ctrl_context;
1363
1364         if (!fusion) {
1365                 megasas_return_cmd(instance, cmd);
1366                 return 1;
1367         }
1368
1369         map = fusion->ld_drv_map[instance->map_id & 1];
1370
1371         num_lds = le16_to_cpu(map->raidMap.ldCount);
1372
1373         dcmd = &cmd->frame->dcmd;
1374
1375         size_sync_info = sizeof(struct MR_LD_TARGET_SYNC) *num_lds;
1376
1377         memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1378
1379         ci = (struct MR_LD_TARGET_SYNC *)
1380           fusion->ld_map[(instance->map_id - 1) & 1];
1381         memset(ci, 0, fusion->max_map_sz);
1382
1383         ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1384
1385         ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1386
1387         for (i = 0; i < num_lds; i++, ld_sync++) {
1388                 raid = MR_LdRaidGet(i, map);
1389                 ld_sync->targetId = MR_GetLDTgtId(i, map);
1390                 ld_sync->seqNum = raid->seqNum;
1391         }
1392
1393         size_map_info = fusion->current_map_sz;
1394
1395         dcmd->cmd = MFI_CMD_DCMD;
1396         dcmd->cmd_status = 0xFF;
1397         dcmd->sge_count = 1;
1398         dcmd->flags = MFI_FRAME_DIR_WRITE;
1399         dcmd->timeout = 0;
1400         dcmd->pad_0 = 0;
1401         dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1402         dcmd->mbox.b[0] = num_lds;
1403         dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1404         dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1405
1406         megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1407
1408         instance->map_update_cmd = cmd;
1409
1410         instance->instancet->issue_dcmd(instance, cmd);
1411
1412         return 0;
1413 }
1414
1415 /*
1416  * meagasas_display_intel_branding - Display branding string
1417  * @instance: per adapter object
1418  *
1419  * Return nothing.
1420  */
1421 static void
1422 megasas_display_intel_branding(struct megasas_instance *instance)
1423 {
1424         if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1425                 return;
1426
1427         switch (instance->pdev->device) {
1428         case PCI_DEVICE_ID_LSI_INVADER:
1429                 switch (instance->pdev->subsystem_device) {
1430                 case MEGARAID_INTEL_RS3DC080_SSDID:
1431                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1432                                 instance->host->host_no,
1433                                 MEGARAID_INTEL_RS3DC080_BRANDING);
1434                         break;
1435                 case MEGARAID_INTEL_RS3DC040_SSDID:
1436                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1437                                 instance->host->host_no,
1438                                 MEGARAID_INTEL_RS3DC040_BRANDING);
1439                         break;
1440                 case MEGARAID_INTEL_RS3SC008_SSDID:
1441                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1442                                 instance->host->host_no,
1443                                 MEGARAID_INTEL_RS3SC008_BRANDING);
1444                         break;
1445                 case MEGARAID_INTEL_RS3MC044_SSDID:
1446                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1447                                 instance->host->host_no,
1448                                 MEGARAID_INTEL_RS3MC044_BRANDING);
1449                         break;
1450                 default:
1451                         break;
1452                 }
1453                 break;
1454         case PCI_DEVICE_ID_LSI_FURY:
1455                 switch (instance->pdev->subsystem_device) {
1456                 case MEGARAID_INTEL_RS3WC080_SSDID:
1457                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1458                                 instance->host->host_no,
1459                                 MEGARAID_INTEL_RS3WC080_BRANDING);
1460                         break;
1461                 case MEGARAID_INTEL_RS3WC040_SSDID:
1462                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1463                                 instance->host->host_no,
1464                                 MEGARAID_INTEL_RS3WC040_BRANDING);
1465                         break;
1466                 default:
1467                         break;
1468                 }
1469                 break;
1470         case PCI_DEVICE_ID_LSI_CUTLASS_52:
1471         case PCI_DEVICE_ID_LSI_CUTLASS_53:
1472                 switch (instance->pdev->subsystem_device) {
1473                 case MEGARAID_INTEL_RMS3BC160_SSDID:
1474                         dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1475                                 instance->host->host_no,
1476                                 MEGARAID_INTEL_RMS3BC160_BRANDING);
1477                         break;
1478                 default:
1479                         break;
1480                 }
1481                 break;
1482         default:
1483                 break;
1484         }
1485 }
1486
1487 /**
1488  * megasas_allocate_raid_maps - Allocate memory for RAID maps
1489  * @instance:                           Adapter soft state
1490  *
1491  * return:                              if success: return 0
1492  *                                      failed:  return -ENOMEM
1493  */
1494 static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
1495 {
1496         struct fusion_context *fusion;
1497         int i = 0;
1498
1499         fusion = instance->ctrl_context;
1500
1501         fusion->drv_map_pages = get_order(fusion->drv_map_sz);
1502
1503         for (i = 0; i < 2; i++) {
1504                 fusion->ld_map[i] = NULL;
1505
1506                 fusion->ld_drv_map[i] = (void *)
1507                         __get_free_pages(__GFP_ZERO | GFP_KERNEL,
1508                                          fusion->drv_map_pages);
1509
1510                 if (!fusion->ld_drv_map[i]) {
1511                         fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);
1512
1513                         if (!fusion->ld_drv_map[i]) {
1514                                 dev_err(&instance->pdev->dev,
1515                                         "Could not allocate memory for local map"
1516                                         " size requested: %d\n",
1517                                         fusion->drv_map_sz);
1518                                 goto ld_drv_map_alloc_fail;
1519                         }
1520                 }
1521         }
1522
1523         for (i = 0; i < 2; i++) {
1524                 fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
1525                                                        fusion->max_map_sz,
1526                                                        &fusion->ld_map_phys[i],
1527                                                        GFP_KERNEL);
1528                 if (!fusion->ld_map[i]) {
1529                         dev_err(&instance->pdev->dev,
1530                                 "Could not allocate memory for map info %s:%d\n",
1531                                 __func__, __LINE__);
1532                         goto ld_map_alloc_fail;
1533                 }
1534         }
1535
1536         return 0;
1537
1538 ld_map_alloc_fail:
1539         for (i = 0; i < 2; i++) {
1540                 if (fusion->ld_map[i])
1541                         dma_free_coherent(&instance->pdev->dev,
1542                                           fusion->max_map_sz,
1543                                           fusion->ld_map[i],
1544                                           fusion->ld_map_phys[i]);
1545         }
1546
1547 ld_drv_map_alloc_fail:
1548         for (i = 0; i < 2; i++) {
1549                 if (fusion->ld_drv_map[i]) {
1550                         if (is_vmalloc_addr(fusion->ld_drv_map[i]))
1551                                 vfree(fusion->ld_drv_map[i]);
1552                         else
1553                                 free_pages((ulong)fusion->ld_drv_map[i],
1554                                            fusion->drv_map_pages);
1555                 }
1556         }
1557
1558         return -ENOMEM;
1559 }
1560
1561 /**
1562  * megasas_configure_queue_sizes -      Calculate size of request desc queue,
1563  *                                      reply desc queue,
1564  *                                      IO request frame queue, set can_queue.
1565  * @instance:                           Adapter soft state
1566  * @return:                             void
1567  */
1568 static inline
1569 void megasas_configure_queue_sizes(struct megasas_instance *instance)
1570 {
1571         struct fusion_context *fusion;
1572         u16 max_cmd;
1573
1574         fusion = instance->ctrl_context;
1575         max_cmd = instance->max_fw_cmds;
1576
1577         if (instance->adapter_type == VENTURA_SERIES)
1578                 instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;
1579         else
1580                 instance->max_mpt_cmds = instance->max_fw_cmds;
1581
1582         instance->max_scsi_cmds = instance->max_fw_cmds -
1583                         (MEGASAS_FUSION_INTERNAL_CMDS +
1584                         MEGASAS_FUSION_IOCTL_CMDS);
1585         instance->cur_can_queue = instance->max_scsi_cmds;
1586         instance->host->can_queue = instance->cur_can_queue;
1587
1588         fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;
1589
1590         fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *
1591                                           instance->max_mpt_cmds;
1592         fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *
1593                                         (fusion->reply_q_depth);
1594         fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1595                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1596                  * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */
1597 }
1598
1599 static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)
1600 {
1601         struct fusion_context *fusion;
1602         struct megasas_cmd *cmd;
1603
1604         fusion = instance->ctrl_context;
1605
1606         cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL);
1607
1608         if (!cmd) {
1609                 dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1610                         __func__, __LINE__);
1611                 return -ENOMEM;
1612         }
1613
1614         cmd->frame = dma_alloc_coherent(&instance->pdev->dev,
1615                                         IOC_INIT_FRAME_SIZE,
1616                                         &cmd->frame_phys_addr, GFP_KERNEL);
1617
1618         if (!cmd->frame) {
1619                 dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1620                         __func__, __LINE__);
1621                 kfree(cmd);
1622                 return -ENOMEM;
1623         }
1624
1625         fusion->ioc_init_cmd = cmd;
1626         return 0;
1627 }
1628
1629 /**
1630  * megasas_free_ioc_init_cmd -  Free IOC INIT command frame
1631  * @instance:           Adapter soft state
1632  */
1633 static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)
1634 {
1635         struct fusion_context *fusion;
1636
1637         fusion = instance->ctrl_context;
1638
1639         if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)
1640                 dma_free_coherent(&instance->pdev->dev,
1641                                   IOC_INIT_FRAME_SIZE,
1642                                   fusion->ioc_init_cmd->frame,
1643                                   fusion->ioc_init_cmd->frame_phys_addr);
1644
1645         if (fusion->ioc_init_cmd)
1646                 kfree(fusion->ioc_init_cmd);
1647 }
1648
1649 /**
1650  * megasas_init_adapter_fusion -        Initializes the FW
1651  * @instance:           Adapter soft state
1652  *
1653  * This is the main function for initializing firmware.
1654  */
1655 u32
1656 megasas_init_adapter_fusion(struct megasas_instance *instance)
1657 {
1658         struct megasas_register_set __iomem *reg_set;
1659         struct fusion_context *fusion;
1660         u32 scratch_pad_2;
1661         int i = 0, count;
1662
1663         fusion = instance->ctrl_context;
1664
1665         reg_set = instance->reg_set;
1666
1667         megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);
1668
1669         /*
1670          * Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames
1671          */
1672         instance->max_mfi_cmds =
1673                 MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1674
1675         megasas_configure_queue_sizes(instance);
1676
1677         scratch_pad_2 = readl(&instance->reg_set->outbound_scratch_pad_2);
1678         /* If scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,
1679          * Firmware support extended IO chain frame which is 4 times more than
1680          * legacy Firmware.
1681          * Legacy Firmware - Frame size is (8 * 128) = 1K
1682          * 1M IO Firmware  - Frame size is (8 * 128 * 4)  = 4K
1683          */
1684         if (scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1685                 instance->max_chain_frame_sz =
1686                         ((scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1687                         MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1688         else
1689                 instance->max_chain_frame_sz =
1690                         ((scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1691                         MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1692
1693         if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1694                 dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1695                         instance->max_chain_frame_sz,
1696                         MEGASAS_CHAIN_FRAME_SZ_MIN);
1697                 instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1698         }
1699
1700         fusion->max_sge_in_main_msg =
1701                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1702                         - offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1703
1704         fusion->max_sge_in_chain =
1705                 instance->max_chain_frame_sz
1706                         / sizeof(union MPI2_SGE_IO_UNION);
1707
1708         instance->max_num_sge =
1709                 rounddown_pow_of_two(fusion->max_sge_in_main_msg
1710                         + fusion->max_sge_in_chain - 2);
1711
1712         /* Used for pass thru MFI frame (DCMD) */
1713         fusion->chain_offset_mfi_pthru =
1714                 offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1715
1716         fusion->chain_offset_io_request =
1717                 (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1718                  sizeof(union MPI2_SGE_IO_UNION))/16;
1719
1720         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1721         for (i = 0 ; i < count; i++)
1722                 fusion->last_reply_idx[i] = 0;
1723
1724         /*
1725          * For fusion adapters, 3 commands for IOCTL and 8 commands
1726          * for driver's internal DCMDs.
1727          */
1728         instance->max_scsi_cmds = instance->max_fw_cmds -
1729                                 (MEGASAS_FUSION_INTERNAL_CMDS +
1730                                 MEGASAS_FUSION_IOCTL_CMDS);
1731         sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1732
1733         if (megasas_alloc_ioc_init_frame(instance))
1734                 return 1;
1735
1736         /*
1737          * Allocate memory for descriptors
1738          * Create a pool of commands
1739          */
1740         if (megasas_alloc_cmds(instance))
1741                 goto fail_alloc_mfi_cmds;
1742         if (megasas_alloc_cmds_fusion(instance))
1743                 goto fail_alloc_cmds;
1744
1745         if (megasas_ioc_init_fusion(instance))
1746                 goto fail_ioc_init;
1747
1748         megasas_display_intel_branding(instance);
1749         if (megasas_get_ctrl_info(instance)) {
1750                 dev_err(&instance->pdev->dev,
1751                         "Could not get controller info. Fail from %s %d\n",
1752                         __func__, __LINE__);
1753                 goto fail_ioc_init;
1754         }
1755
1756         instance->flag_ieee = 1;
1757         instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;
1758         fusion->fast_path_io = 0;
1759
1760         if (megasas_allocate_raid_maps(instance))
1761                 goto fail_ioc_init;
1762
1763         if (!megasas_get_map_info(instance))
1764                 megasas_sync_map_info(instance);
1765
1766         return 0;
1767
1768 fail_ioc_init:
1769         megasas_free_cmds_fusion(instance);
1770 fail_alloc_cmds:
1771         megasas_free_cmds(instance);
1772 fail_alloc_mfi_cmds:
1773         megasas_free_ioc_init_cmd(instance);
1774         return 1;
1775 }
1776
1777 /**
1778  * map_cmd_status -     Maps FW cmd status to OS cmd status
1779  * @cmd :               Pointer to cmd
1780  * @status :            status of cmd returned by FW
1781  * @ext_status :        ext status of cmd returned by FW
1782  */
1783
1784 void
1785 map_cmd_status(struct fusion_context *fusion,
1786                 struct scsi_cmnd *scmd, u8 status, u8 ext_status,
1787                 u32 data_length, u8 *sense)
1788 {
1789         u8 cmd_type;
1790         int resid;
1791
1792         cmd_type = megasas_cmd_type(scmd);
1793         switch (status) {
1794
1795         case MFI_STAT_OK:
1796                 scmd->result = DID_OK << 16;
1797                 break;
1798
1799         case MFI_STAT_SCSI_IO_FAILED:
1800         case MFI_STAT_LD_INIT_IN_PROGRESS:
1801                 scmd->result = (DID_ERROR << 16) | ext_status;
1802                 break;
1803
1804         case MFI_STAT_SCSI_DONE_WITH_ERROR:
1805
1806                 scmd->result = (DID_OK << 16) | ext_status;
1807                 if (ext_status == SAM_STAT_CHECK_CONDITION) {
1808                         memset(scmd->sense_buffer, 0,
1809                                SCSI_SENSE_BUFFERSIZE);
1810                         memcpy(scmd->sense_buffer, sense,
1811                                SCSI_SENSE_BUFFERSIZE);
1812                         scmd->result |= DRIVER_SENSE << 24;
1813                 }
1814
1815                 /*
1816                  * If the  IO request is partially completed, then MR FW will
1817                  * update "io_request->DataLength" field with actual number of
1818                  * bytes transferred.Driver will set residual bytes count in
1819                  * SCSI command structure.
1820                  */
1821                 resid = (scsi_bufflen(scmd) - data_length);
1822                 scsi_set_resid(scmd, resid);
1823
1824                 if (resid &&
1825                         ((cmd_type == READ_WRITE_LDIO) ||
1826                         (cmd_type == READ_WRITE_SYSPDIO)))
1827                         scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
1828                                 " requested/completed 0x%x/0x%x\n",
1829                                 status, scsi_bufflen(scmd), data_length);
1830                 break;
1831
1832         case MFI_STAT_LD_OFFLINE:
1833         case MFI_STAT_DEVICE_NOT_FOUND:
1834                 scmd->result = DID_BAD_TARGET << 16;
1835                 break;
1836         case MFI_STAT_CONFIG_SEQ_MISMATCH:
1837                 scmd->result = DID_IMM_RETRY << 16;
1838                 break;
1839         default:
1840                 scmd->result = DID_ERROR << 16;
1841                 break;
1842         }
1843 }
1844
1845 /**
1846  * megasas_is_prp_possible -
1847  * Checks if native NVMe PRPs can be built for the IO
1848  *
1849  * @instance:           Adapter soft state
1850  * @scmd:               SCSI command from the mid-layer
1851  * @sge_count:          scatter gather element count.
1852  *
1853  * Returns:             true: PRPs can be built
1854  *                      false: IEEE SGLs needs to be built
1855  */
1856 static bool
1857 megasas_is_prp_possible(struct megasas_instance *instance,
1858                         struct scsi_cmnd *scmd, int sge_count)
1859 {
1860         struct fusion_context *fusion;
1861         int i;
1862         u32 data_length = 0;
1863         struct scatterlist *sg_scmd;
1864         bool build_prp = false;
1865         u32 mr_nvme_pg_size;
1866
1867         mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
1868                                 MR_DEFAULT_NVME_PAGE_SIZE);
1869         fusion = instance->ctrl_context;
1870         data_length = scsi_bufflen(scmd);
1871         sg_scmd = scsi_sglist(scmd);
1872
1873         /*
1874          * NVMe uses one PRP for each page (or part of a page)
1875          * look at the data length - if 4 pages or less then IEEE is OK
1876          * if  > 5 pages then we need to build a native SGL
1877          * if > 4 and <= 5 pages, then check physical address of 1st SG entry
1878          * if this first size in the page is >= the residual beyond 4 pages
1879          * then use IEEE, otherwise use native SGL
1880          */
1881
1882         if (data_length > (mr_nvme_pg_size * 5)) {
1883                 build_prp = true;
1884         } else if ((data_length > (mr_nvme_pg_size * 4)) &&
1885                         (data_length <= (mr_nvme_pg_size * 5)))  {
1886                 /* check if 1st SG entry size is < residual beyond 4 pages */
1887                 if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
1888                         build_prp = true;
1889         }
1890
1891 /*
1892  * Below code detects gaps/holes in IO data buffers.
1893  * What does holes/gaps mean?
1894  * Any SGE except first one in a SGL starts at non NVME page size
1895  * aligned address OR Any SGE except last one in a SGL ends at
1896  * non NVME page size boundary.
1897  *
1898  * Driver has already informed block layer by setting boundary rules for
1899  * bio merging done at NVME page size boundary calling kernel API
1900  * blk_queue_virt_boundary inside slave_config.
1901  * Still there is possibility of IO coming with holes to driver because of
1902  * IO merging done by IO scheduler.
1903  *
1904  * With SCSI BLK MQ enabled, there will be no IO with holes as there is no
1905  * IO scheduling so no IO merging.
1906  *
1907  * With SCSI BLK MQ disabled, IO scheduler may attempt to merge IOs and
1908  * then sending IOs with holes.
1909  *
1910  * Though driver can request block layer to disable IO merging by calling-
1911  * queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, sdev->request_queue) but
1912  * user may tune sysfs parameter- nomerges again to 0 or 1.
1913  *
1914  * If in future IO scheduling is enabled with SCSI BLK MQ,
1915  * this algorithm to detect holes will be required in driver
1916  * for SCSI BLK MQ enabled case as well.
1917  *
1918  *
1919  */
1920         scsi_for_each_sg(scmd, sg_scmd, sge_count, i) {
1921                 if ((i != 0) && (i != (sge_count - 1))) {
1922                         if (mega_mod64(sg_dma_len(sg_scmd), mr_nvme_pg_size) ||
1923                             mega_mod64(sg_dma_address(sg_scmd),
1924                                        mr_nvme_pg_size)) {
1925                                 build_prp = false;
1926                                 atomic_inc(&instance->sge_holes_type1);
1927                                 break;
1928                         }
1929                 }
1930
1931                 if ((sge_count > 1) && (i == 0)) {
1932                         if ((mega_mod64((sg_dma_address(sg_scmd) +
1933                                         sg_dma_len(sg_scmd)),
1934                                         mr_nvme_pg_size))) {
1935                                 build_prp = false;
1936                                 atomic_inc(&instance->sge_holes_type2);
1937                                 break;
1938                         }
1939                 }
1940
1941                 if ((sge_count > 1) && (i == (sge_count - 1))) {
1942                         if (mega_mod64(sg_dma_address(sg_scmd),
1943                                        mr_nvme_pg_size)) {
1944                                 build_prp = false;
1945                                 atomic_inc(&instance->sge_holes_type3);
1946                                 break;
1947                         }
1948                 }
1949         }
1950
1951         return build_prp;
1952 }
1953
1954 /**
1955  * megasas_make_prp_nvme -
1956  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
1957  *
1958  * @instance:           Adapter soft state
1959  * @scmd:               SCSI command from the mid-layer
1960  * @sgl_ptr:            SGL to be filled in
1961  * @cmd:                Fusion command frame
1962  * @sge_count:          scatter gather element count.
1963  *
1964  * Returns:             true: PRPs are built
1965  *                      false: IEEE SGLs needs to be built
1966  */
1967 static bool
1968 megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
1969                       struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
1970                       struct megasas_cmd_fusion *cmd, int sge_count)
1971 {
1972         int sge_len, offset, num_prp_in_chain = 0;
1973         struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
1974         u64 *ptr_sgl;
1975         dma_addr_t ptr_sgl_phys;
1976         u64 sge_addr;
1977         u32 page_mask, page_mask_result;
1978         struct scatterlist *sg_scmd;
1979         u32 first_prp_len;
1980         bool build_prp = false;
1981         int data_len = scsi_bufflen(scmd);
1982         struct fusion_context *fusion;
1983         u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
1984                                         MR_DEFAULT_NVME_PAGE_SIZE);
1985
1986         fusion = instance->ctrl_context;
1987
1988         build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
1989
1990         if (!build_prp)
1991                 return false;
1992
1993         /*
1994          * Nvme has a very convoluted prp format.  One prp is required
1995          * for each page or partial page. Driver need to split up OS sg_list
1996          * entries if it is longer than one page or cross a page
1997          * boundary.  Driver also have to insert a PRP list pointer entry as
1998          * the last entry in each physical page of the PRP list.
1999          *
2000          * NOTE: The first PRP "entry" is actually placed in the first
2001          * SGL entry in the main message as IEEE 64 format.  The 2nd
2002          * entry in the main message is the chain element, and the rest
2003          * of the PRP entries are built in the contiguous pcie buffer.
2004          */
2005         page_mask = mr_nvme_pg_size - 1;
2006         ptr_sgl = (u64 *)cmd->sg_frame;
2007         ptr_sgl_phys = cmd->sg_frame_phys_addr;
2008         memset(ptr_sgl, 0, instance->max_chain_frame_sz);
2009
2010         /* Build chain frame element which holds all prps except first*/
2011         main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
2012             ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
2013
2014         main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
2015         main_chain_element->NextChainOffset = 0;
2016         main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2017                                         IEEE_SGE_FLAGS_SYSTEM_ADDR |
2018                                         MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2019
2020         /* Build first prp, sge need not to be page aligned*/
2021         ptr_first_sgl = sgl_ptr;
2022         sg_scmd = scsi_sglist(scmd);
2023         sge_addr = sg_dma_address(sg_scmd);
2024         sge_len = sg_dma_len(sg_scmd);
2025
2026         offset = (u32)(sge_addr & page_mask);
2027         first_prp_len = mr_nvme_pg_size - offset;
2028
2029         ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2030         ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2031
2032         data_len -= first_prp_len;
2033
2034         if (sge_len > first_prp_len) {
2035                 sge_addr += first_prp_len;
2036                 sge_len -= first_prp_len;
2037         } else if (sge_len == first_prp_len) {
2038                 sg_scmd = sg_next(sg_scmd);
2039                 sge_addr = sg_dma_address(sg_scmd);
2040                 sge_len = sg_dma_len(sg_scmd);
2041         }
2042
2043         for (;;) {
2044                 offset = (u32)(sge_addr & page_mask);
2045
2046                 /* Put PRP pointer due to page boundary*/
2047                 page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
2048                 if (unlikely(!page_mask_result)) {
2049                         scmd_printk(KERN_NOTICE,
2050                                     scmd, "page boundary ptr_sgl: 0x%p\n",
2051                                     ptr_sgl);
2052                         ptr_sgl_phys += 8;
2053                         *ptr_sgl = cpu_to_le64(ptr_sgl_phys);
2054                         ptr_sgl++;
2055                         num_prp_in_chain++;
2056                 }
2057
2058                 *ptr_sgl = cpu_to_le64(sge_addr);
2059                 ptr_sgl++;
2060                 ptr_sgl_phys += 8;
2061                 num_prp_in_chain++;
2062
2063                 sge_addr += mr_nvme_pg_size;
2064                 sge_len -= mr_nvme_pg_size;
2065                 data_len -= mr_nvme_pg_size;
2066
2067                 if (data_len <= 0)
2068                         break;
2069
2070                 if (sge_len > 0)
2071                         continue;
2072
2073                 sg_scmd = sg_next(sg_scmd);
2074                 sge_addr = sg_dma_address(sg_scmd);
2075                 sge_len = sg_dma_len(sg_scmd);
2076         }
2077
2078         main_chain_element->Length =
2079                         cpu_to_le32(num_prp_in_chain * sizeof(u64));
2080
2081         atomic_inc(&instance->prp_sgl);
2082         return build_prp;
2083 }
2084
2085 /**
2086  * megasas_make_sgl_fusion -    Prepares 32-bit SGL
2087  * @instance:           Adapter soft state
2088  * @scp:                SCSI command from the mid-layer
2089  * @sgl_ptr:            SGL to be filled in
2090  * @cmd:                cmd we are working on
2091  * @sge_count           sge count
2092  *
2093  */
2094 static void
2095 megasas_make_sgl_fusion(struct megasas_instance *instance,
2096                         struct scsi_cmnd *scp,
2097                         struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2098                         struct megasas_cmd_fusion *cmd, int sge_count)
2099 {
2100         int i, sg_processed;
2101         struct scatterlist *os_sgl;
2102         struct fusion_context *fusion;
2103
2104         fusion = instance->ctrl_context;
2105
2106         if (instance->adapter_type >= INVADER_SERIES) {
2107                 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
2108                 sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
2109                 sgl_ptr_end->Flags = 0;
2110         }
2111
2112         scsi_for_each_sg(scp, os_sgl, sge_count, i) {
2113                 sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
2114                 sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
2115                 sgl_ptr->Flags = 0;
2116                 if (instance->adapter_type >= INVADER_SERIES)
2117                         if (i == sge_count - 1)
2118                                 sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
2119                 sgl_ptr++;
2120                 sg_processed = i + 1;
2121
2122                 if ((sg_processed ==  (fusion->max_sge_in_main_msg - 1)) &&
2123                     (sge_count > fusion->max_sge_in_main_msg)) {
2124
2125                         struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
2126                         if (instance->adapter_type >= INVADER_SERIES) {
2127                                 if ((le16_to_cpu(cmd->io_request->IoFlags) &
2128                                         MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
2129                                         MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
2130                                         cmd->io_request->ChainOffset =
2131                                                 fusion->
2132                                                 chain_offset_io_request;
2133                                 else
2134                                         cmd->io_request->ChainOffset = 0;
2135                         } else
2136                                 cmd->io_request->ChainOffset =
2137                                         fusion->chain_offset_io_request;
2138
2139                         sg_chain = sgl_ptr;
2140                         /* Prepare chain element */
2141                         sg_chain->NextChainOffset = 0;
2142                         if (instance->adapter_type >= INVADER_SERIES)
2143                                 sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
2144                         else
2145                                 sg_chain->Flags =
2146                                         (IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2147                                          MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
2148                         sg_chain->Length =  cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
2149                         sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
2150
2151                         sgl_ptr =
2152                           (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
2153                         memset(sgl_ptr, 0, instance->max_chain_frame_sz);
2154                 }
2155         }
2156         atomic_inc(&instance->ieee_sgl);
2157 }
2158
2159 /**
2160  * megasas_make_sgl -   Build Scatter Gather List(SGLs)
2161  * @scp:                SCSI command pointer
2162  * @instance:           Soft instance of controller
2163  * @cmd:                Fusion command pointer
2164  *
2165  * This function will build sgls based on device type.
2166  * For nvme drives, there is different way of building sgls in nvme native
2167  * format- PRPs(Physical Region Page).
2168  *
2169  * Returns the number of sg lists actually used, zero if the sg lists
2170  * is NULL, or -ENOMEM if the mapping failed
2171  */
2172 static
2173 int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
2174                      struct megasas_cmd_fusion *cmd)
2175 {
2176         int sge_count;
2177         bool build_prp = false;
2178         struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
2179
2180         sge_count = scsi_dma_map(scp);
2181
2182         if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
2183                 return sge_count;
2184
2185         sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
2186         if ((le16_to_cpu(cmd->io_request->IoFlags) &
2187             MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
2188             (cmd->pd_interface == NVME_PD))
2189                 build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,
2190                                                   cmd, sge_count);
2191
2192         if (!build_prp)
2193                 megasas_make_sgl_fusion(instance, scp, sgl_chain64,
2194                                         cmd, sge_count);
2195
2196         return sge_count;
2197 }
2198
2199 /**
2200  * megasas_set_pd_lba - Sets PD LBA
2201  * @cdb:                CDB
2202  * @cdb_len:            cdb length
2203  * @start_blk:          Start block of IO
2204  *
2205  * Used to set the PD LBA in CDB for FP IOs
2206  */
2207 void
2208 megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
2209                    struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
2210                    struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
2211 {
2212         struct MR_LD_RAID *raid;
2213         u16 ld;
2214         u64 start_blk = io_info->pdBlock;
2215         u8 *cdb = io_request->CDB.CDB32;
2216         u32 num_blocks = io_info->numBlocks;
2217         u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
2218
2219         /* Check if T10 PI (DIF) is enabled for this LD */
2220         ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);
2221         raid = MR_LdRaidGet(ld, local_map_ptr);
2222         if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
2223                 memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2224                 cdb[0] =  MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
2225                 cdb[7] =  MEGASAS_SCSI_ADDL_CDB_LEN;
2226
2227                 if (scp->sc_data_direction == PCI_DMA_FROMDEVICE)
2228                         cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
2229                 else
2230                         cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
2231                 cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
2232
2233                 /* LBA */
2234                 cdb[12] = (u8)((start_blk >> 56) & 0xff);
2235                 cdb[13] = (u8)((start_blk >> 48) & 0xff);
2236                 cdb[14] = (u8)((start_blk >> 40) & 0xff);
2237                 cdb[15] = (u8)((start_blk >> 32) & 0xff);
2238                 cdb[16] = (u8)((start_blk >> 24) & 0xff);
2239                 cdb[17] = (u8)((start_blk >> 16) & 0xff);
2240                 cdb[18] = (u8)((start_blk >> 8) & 0xff);
2241                 cdb[19] = (u8)(start_blk & 0xff);
2242
2243                 /* Logical block reference tag */
2244                 io_request->CDB.EEDP32.PrimaryReferenceTag =
2245                         cpu_to_be32(ref_tag);
2246                 io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
2247                 io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */
2248
2249                 /* Transfer length */
2250                 cdb[28] = (u8)((num_blocks >> 24) & 0xff);
2251                 cdb[29] = (u8)((num_blocks >> 16) & 0xff);
2252                 cdb[30] = (u8)((num_blocks >> 8) & 0xff);
2253                 cdb[31] = (u8)(num_blocks & 0xff);
2254
2255                 /* set SCSI IO EEDPFlags */
2256                 if (scp->sc_data_direction == PCI_DMA_FROMDEVICE) {
2257                         io_request->EEDPFlags = cpu_to_le16(
2258                                 MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG  |
2259                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2260                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
2261                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
2262                                 MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
2263                                 MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
2264                 } else {
2265                         io_request->EEDPFlags = cpu_to_le16(
2266                                 MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2267                                 MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
2268                 }
2269                 io_request->Control |= cpu_to_le32((0x4 << 26));
2270                 io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
2271         } else {
2272                 /* Some drives don't support 16/12 byte CDB's, convert to 10 */
2273                 if (((cdb_len == 12) || (cdb_len == 16)) &&
2274                     (start_blk <= 0xffffffff)) {
2275                         if (cdb_len == 16) {
2276                                 opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
2277                                 flagvals = cdb[1];
2278                                 groupnum = cdb[14];
2279                                 control = cdb[15];
2280                         } else {
2281                                 opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
2282                                 flagvals = cdb[1];
2283                                 groupnum = cdb[10];
2284                                 control = cdb[11];
2285                         }
2286
2287                         memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2288
2289                         cdb[0] = opcode;
2290                         cdb[1] = flagvals;
2291                         cdb[6] = groupnum;
2292                         cdb[9] = control;
2293
2294                         /* Transfer length */
2295                         cdb[8] = (u8)(num_blocks & 0xff);
2296                         cdb[7] = (u8)((num_blocks >> 8) & 0xff);
2297
2298                         io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */
2299                         cdb_len = 10;
2300                 } else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
2301                         /* Convert to 16 byte CDB for large LBA's */
2302                         switch (cdb_len) {
2303                         case 6:
2304                                 opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
2305                                 control = cdb[5];
2306                                 break;
2307                         case 10:
2308                                 opcode =
2309                                         cdb[0] == READ_10 ? READ_16 : WRITE_16;
2310                                 flagvals = cdb[1];
2311                                 groupnum = cdb[6];
2312                                 control = cdb[9];
2313                                 break;
2314                         case 12:
2315                                 opcode =
2316                                         cdb[0] == READ_12 ? READ_16 : WRITE_16;
2317                                 flagvals = cdb[1];
2318                                 groupnum = cdb[10];
2319                                 control = cdb[11];
2320                                 break;
2321                         }
2322
2323                         memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2324
2325                         cdb[0] = opcode;
2326                         cdb[1] = flagvals;
2327                         cdb[14] = groupnum;
2328                         cdb[15] = control;
2329
2330                         /* Transfer length */
2331                         cdb[13] = (u8)(num_blocks & 0xff);
2332                         cdb[12] = (u8)((num_blocks >> 8) & 0xff);
2333                         cdb[11] = (u8)((num_blocks >> 16) & 0xff);
2334                         cdb[10] = (u8)((num_blocks >> 24) & 0xff);
2335
2336                         io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */
2337                         cdb_len = 16;
2338                 }
2339
2340                 /* Normal case, just load LBA here */
2341                 switch (cdb_len) {
2342                 case 6:
2343                 {
2344                         u8 val = cdb[1] & 0xE0;
2345                         cdb[3] = (u8)(start_blk & 0xff);
2346                         cdb[2] = (u8)((start_blk >> 8) & 0xff);
2347                         cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
2348                         break;
2349                 }
2350                 case 10:
2351                         cdb[5] = (u8)(start_blk & 0xff);
2352                         cdb[4] = (u8)((start_blk >> 8) & 0xff);
2353                         cdb[3] = (u8)((start_blk >> 16) & 0xff);
2354                         cdb[2] = (u8)((start_blk >> 24) & 0xff);
2355                         break;
2356                 case 12:
2357                         cdb[5]    = (u8)(start_blk & 0xff);
2358                         cdb[4]    = (u8)((start_blk >> 8) & 0xff);
2359                         cdb[3]    = (u8)((start_blk >> 16) & 0xff);
2360                         cdb[2]    = (u8)((start_blk >> 24) & 0xff);
2361                         break;
2362                 case 16:
2363                         cdb[9]    = (u8)(start_blk & 0xff);
2364                         cdb[8]    = (u8)((start_blk >> 8) & 0xff);
2365                         cdb[7]    = (u8)((start_blk >> 16) & 0xff);
2366                         cdb[6]    = (u8)((start_blk >> 24) & 0xff);
2367                         cdb[5]    = (u8)((start_blk >> 32) & 0xff);
2368                         cdb[4]    = (u8)((start_blk >> 40) & 0xff);
2369                         cdb[3]    = (u8)((start_blk >> 48) & 0xff);
2370                         cdb[2]    = (u8)((start_blk >> 56) & 0xff);
2371                         break;
2372                 }
2373         }
2374 }
2375
2376 /**
2377  * megasas_stream_detect -      stream detection on read and and write IOs
2378  * @instance:           Adapter soft state
2379  * @cmd:                    Command to be prepared
2380  * @io_info:            IO Request info
2381  *
2382  */
2383
2384 /** stream detection on read and and write IOs */
2385 static void megasas_stream_detect(struct megasas_instance *instance,
2386                                   struct megasas_cmd_fusion *cmd,
2387                                   struct IO_REQUEST_INFO *io_info)
2388 {
2389         struct fusion_context *fusion = instance->ctrl_context;
2390         u32 device_id = io_info->ldTgtId;
2391         struct LD_STREAM_DETECT *current_ld_sd
2392                 = fusion->stream_detect_by_ld[device_id];
2393         u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;
2394         u32 shifted_values, unshifted_values;
2395         u32 index_value_mask, shifted_values_mask;
2396         int i;
2397         bool is_read_ahead = false;
2398         struct STREAM_DETECT *current_sd;
2399         /* find possible stream */
2400         for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2401                 stream_num = (*track_stream >>
2402                         (i * BITS_PER_INDEX_STREAM)) &
2403                         STREAM_MASK;
2404                 current_sd = &current_ld_sd->stream_track[stream_num];
2405                 /* if we found a stream, update the raid
2406                  *  context and also update the mruBitMap
2407                  */
2408                 /*      boundary condition */
2409                 if ((current_sd->next_seq_lba) &&
2410                     (io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2411                     (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2412                     (current_sd->is_read == io_info->isRead)) {
2413
2414                         if ((io_info->ldStartBlock != current_sd->next_seq_lba) &&
2415                             ((!io_info->isRead) || (!is_read_ahead)))
2416                                 /*
2417                                  * Once the API availible we need to change this.
2418                                  * At this point we are not allowing any gap
2419                                  */
2420                                 continue;
2421
2422                         SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2423                         current_sd->next_seq_lba =
2424                         io_info->ldStartBlock + io_info->numBlocks;
2425                         /*
2426                          *      update the mruBitMap LRU
2427                          */
2428                         shifted_values_mask =
2429                                 (1 <<  i * BITS_PER_INDEX_STREAM) - 1;
2430                         shifted_values = ((*track_stream & shifted_values_mask)
2431                                                 << BITS_PER_INDEX_STREAM);
2432                         index_value_mask =
2433                                 STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2434                         unshifted_values =
2435                                 *track_stream & ~(shifted_values_mask |
2436                                 index_value_mask);
2437                         *track_stream =
2438                                 unshifted_values | shifted_values | stream_num;
2439                         return;
2440                 }
2441         }
2442         /*
2443          * if we did not find any stream, create a new one
2444          * from the least recently used
2445          */
2446         stream_num = (*track_stream >>
2447                 ((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2448                 STREAM_MASK;
2449         current_sd = &current_ld_sd->stream_track[stream_num];
2450         current_sd->is_read = io_info->isRead;
2451         current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2452         *track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2453         return;
2454 }
2455
2456 /**
2457  * megasas_set_raidflag_cpu_affinity - This function sets the cpu
2458  * affinity (cpu of the controller) and raid_flags in the raid context
2459  * based on IO type.
2460  *
2461  * @praid_context:      IO RAID context
2462  * @raid:               LD raid map
2463  * @fp_possible:        Is fast path possible?
2464  * @is_read:            Is read IO?
2465  *
2466  */
2467 static void
2468 megasas_set_raidflag_cpu_affinity(union RAID_CONTEXT_UNION *praid_context,
2469                                   struct MR_LD_RAID *raid, bool fp_possible,
2470                                   u8 is_read, u32 scsi_buff_len)
2471 {
2472         u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2473         struct RAID_CONTEXT_G35 *rctx_g35;
2474
2475         rctx_g35 = &praid_context->raid_context_g35;
2476         if (fp_possible) {
2477                 if (is_read) {
2478                         if ((raid->cpuAffinity.pdRead.cpu0) &&
2479                             (raid->cpuAffinity.pdRead.cpu1))
2480                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2481                         else if (raid->cpuAffinity.pdRead.cpu1)
2482                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2483                 } else {
2484                         if ((raid->cpuAffinity.pdWrite.cpu0) &&
2485                             (raid->cpuAffinity.pdWrite.cpu1))
2486                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2487                         else if (raid->cpuAffinity.pdWrite.cpu1)
2488                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2489                         /* Fast path cache by pass capable R0/R1 VD */
2490                         if ((raid->level <= 1) &&
2491                             (raid->capability.fp_cache_bypass_capable)) {
2492                                 rctx_g35->routing_flags |=
2493                                         (1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2494                                 rctx_g35->raid_flags =
2495                                         (MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2496                                         << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2497                         }
2498                 }
2499         } else {
2500                 if (is_read) {
2501                         if ((raid->cpuAffinity.ldRead.cpu0) &&
2502                             (raid->cpuAffinity.ldRead.cpu1))
2503                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2504                         else if (raid->cpuAffinity.ldRead.cpu1)
2505                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2506                 } else {
2507                         if ((raid->cpuAffinity.ldWrite.cpu0) &&
2508                             (raid->cpuAffinity.ldWrite.cpu1))
2509                                 cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2510                         else if (raid->cpuAffinity.ldWrite.cpu1)
2511                                 cpu_sel = MR_RAID_CTX_CPUSEL_1;
2512
2513                         if (is_stream_detected(rctx_g35) &&
2514                             ((raid->level == 5) || (raid->level == 6)) &&
2515                             (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2516                             (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2517                                 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2518                 }
2519         }
2520
2521         rctx_g35->routing_flags |=
2522                 (cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2523
2524         /* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2525          * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.
2526          * IO Subtype is not bitmap.
2527          */
2528         if ((raid->level == 1) && (!is_read)) {
2529                 if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2530                         praid_context->raid_context_g35.raid_flags =
2531                                 (MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2532                                 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2533         }
2534 }
2535
2536 /**
2537  * megasas_build_ldio_fusion -  Prepares IOs to devices
2538  * @instance:           Adapter soft state
2539  * @scp:                SCSI command
2540  * @cmd:                Command to be prepared
2541  *
2542  * Prepares the io_request and chain elements (sg_frame) for IO
2543  * The IO can be for PD (Fast Path) or LD
2544  */
2545 void
2546 megasas_build_ldio_fusion(struct megasas_instance *instance,
2547                           struct scsi_cmnd *scp,
2548                           struct megasas_cmd_fusion *cmd)
2549 {
2550         bool fp_possible;
2551         u16 ld;
2552         u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2553         u32 scsi_buff_len;
2554         struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2555         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
2556         struct IO_REQUEST_INFO io_info;
2557         struct fusion_context *fusion;
2558         struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2559         u8 *raidLUN;
2560         unsigned long spinlock_flags;
2561         union RAID_CONTEXT_UNION *praid_context;
2562         struct MR_LD_RAID *raid = NULL;
2563         struct MR_PRIV_DEVICE *mrdev_priv;
2564
2565         device_id = MEGASAS_DEV_INDEX(scp);
2566
2567         fusion = instance->ctrl_context;
2568
2569         io_request = cmd->io_request;
2570         io_request->RaidContext.raid_context.virtual_disk_tgt_id =
2571                 cpu_to_le16(device_id);
2572         io_request->RaidContext.raid_context.status = 0;
2573         io_request->RaidContext.raid_context.ex_status = 0;
2574
2575         req_desc = (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)cmd->request_desc;
2576
2577         start_lba_lo = 0;
2578         start_lba_hi = 0;
2579         fp_possible = false;
2580
2581         /*
2582          * 6-byte READ(0x08) or WRITE(0x0A) cdb
2583          */
2584         if (scp->cmd_len == 6) {
2585                 datalength = (u32) scp->cmnd[4];
2586                 start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2587                         ((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2588
2589                 start_lba_lo &= 0x1FFFFF;
2590         }
2591
2592         /*
2593          * 10-byte READ(0x28) or WRITE(0x2A) cdb
2594          */
2595         else if (scp->cmd_len == 10) {
2596                 datalength = (u32) scp->cmnd[8] |
2597                         ((u32) scp->cmnd[7] << 8);
2598                 start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2599                         ((u32) scp->cmnd[3] << 16) |
2600                         ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2601         }
2602
2603         /*
2604          * 12-byte READ(0xA8) or WRITE(0xAA) cdb
2605          */
2606         else if (scp->cmd_len == 12) {
2607                 datalength = ((u32) scp->cmnd[6] << 24) |
2608                         ((u32) scp->cmnd[7] << 16) |
2609                         ((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2610                 start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2611                         ((u32) scp->cmnd[3] << 16) |
2612                         ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2613         }
2614
2615         /*
2616          * 16-byte READ(0x88) or WRITE(0x8A) cdb
2617          */
2618         else if (scp->cmd_len == 16) {
2619                 datalength = ((u32) scp->cmnd[10] << 24) |
2620                         ((u32) scp->cmnd[11] << 16) |
2621                         ((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2622                 start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2623                         ((u32) scp->cmnd[7] << 16) |
2624                         ((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2625
2626                 start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2627                         ((u32) scp->cmnd[3] << 16) |
2628                         ((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2629         }
2630
2631         memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2632         io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2633         io_info.numBlocks = datalength;
2634         io_info.ldTgtId = device_id;
2635         io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2636         scsi_buff_len = scsi_bufflen(scp);
2637         io_request->DataLength = cpu_to_le32(scsi_buff_len);
2638
2639         if (scp->sc_data_direction == PCI_DMA_FROMDEVICE)
2640                 io_info.isRead = 1;
2641
2642         local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2643         ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2644
2645         if (ld < instance->fw_supported_vd_count)
2646                 raid = MR_LdRaidGet(ld, local_map_ptr);
2647
2648         if (!raid || (!fusion->fast_path_io)) {
2649                 io_request->RaidContext.raid_context.reg_lock_flags  = 0;
2650                 fp_possible = false;
2651         } else {
2652                 if (MR_BuildRaidContext(instance, &io_info,
2653                                         &io_request->RaidContext.raid_context,
2654                                         local_map_ptr, &raidLUN))
2655                         fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2656         }
2657
2658         /* Use raw_smp_processor_id() for now until cmd->request->cpu is CPU
2659            id by default, not CPU group id, otherwise all MSI-X queues won't
2660            be utilized */
2661         cmd->request_desc->SCSIIO.MSIxIndex = instance->msix_vectors ?
2662                 raw_smp_processor_id() % instance->msix_vectors : 0;
2663
2664         praid_context = &io_request->RaidContext;
2665
2666         if (instance->adapter_type == VENTURA_SERIES) {
2667                 /* FP for Optimal raid level 1.
2668                  * All large RAID-1 writes (> 32 KiB, both WT and WB modes)
2669                  * are built by the driver as LD I/Os.
2670                  * All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os
2671                  * (there is never a reason to process these as buffered writes)
2672                  * All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os
2673                  * with the SLD bit asserted.
2674                  */
2675                 if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2676                         mrdev_priv = scp->device->hostdata;
2677
2678                         if (atomic_inc_return(&instance->fw_outstanding) >
2679                                 (instance->host->can_queue)) {
2680                                 fp_possible = false;
2681                                 atomic_dec(&instance->fw_outstanding);
2682                         } else if ((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2683                                    (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0)) {
2684                                 fp_possible = false;
2685                                 atomic_dec(&instance->fw_outstanding);
2686                                 if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2687                                         atomic_set(&mrdev_priv->r1_ldio_hint,
2688                                                    instance->r1_ldio_hint_default);
2689                         }
2690                 }
2691
2692                 if (!fp_possible ||
2693                     (io_info.isRead && io_info.ra_capable)) {
2694                         spin_lock_irqsave(&instance->stream_lock,
2695                                           spinlock_flags);
2696                         megasas_stream_detect(instance, cmd, &io_info);
2697                         spin_unlock_irqrestore(&instance->stream_lock,
2698                                                spinlock_flags);
2699                         /* In ventura if stream detected for a read and it is
2700                          * read ahead capable make this IO as LDIO
2701                          */
2702                         if (is_stream_detected(&io_request->RaidContext.raid_context_g35))
2703                                 fp_possible = false;
2704                 }
2705
2706                 /* If raid is NULL, set CPU affinity to default CPU0 */
2707                 if (raid)
2708                         megasas_set_raidflag_cpu_affinity(praid_context,
2709                                 raid, fp_possible, io_info.isRead,
2710                                 scsi_buff_len);
2711                 else
2712                         praid_context->raid_context_g35.routing_flags |=
2713                                 (MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2714         }
2715
2716         if (fp_possible) {
2717                 megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
2718                                    local_map_ptr, start_lba_lo);
2719                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2720                 cmd->request_desc->SCSIIO.RequestFlags =
2721                         (MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2722                          << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2723                 if (instance->adapter_type == INVADER_SERIES) {
2724                         if (io_request->RaidContext.raid_context.reg_lock_flags ==
2725                             REGION_TYPE_UNUSED)
2726                                 cmd->request_desc->SCSIIO.RequestFlags =
2727                                         (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2728                                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2729                         io_request->RaidContext.raid_context.type
2730                                 = MPI2_TYPE_CUDA;
2731                         io_request->RaidContext.raid_context.nseg = 0x1;
2732                         io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2733                         io_request->RaidContext.raid_context.reg_lock_flags |=
2734                           (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2735                            MR_RL_FLAGS_SEQ_NUM_ENABLE);
2736                 } else if (instance->adapter_type == VENTURA_SERIES) {
2737                         io_request->RaidContext.raid_context_g35.nseg_type |=
2738                                                 (1 << RAID_CONTEXT_NSEG_SHIFT);
2739                         io_request->RaidContext.raid_context_g35.nseg_type |=
2740                                                 (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2741                         io_request->RaidContext.raid_context_g35.routing_flags |=
2742                                                 (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2743                         io_request->IoFlags |=
2744                                 cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2745                 }
2746                 if (fusion->load_balance_info &&
2747                         (fusion->load_balance_info[device_id].loadBalanceFlag) &&
2748                         (io_info.isRead)) {
2749                         io_info.devHandle =
2750                                 get_updated_dev_handle(instance,
2751                                         &fusion->load_balance_info[device_id],
2752                                         &io_info, local_map_ptr);
2753                         scp->SCp.Status |= MEGASAS_LOAD_BALANCE_FLAG;
2754                         cmd->pd_r1_lb = io_info.pd_after_lb;
2755                         if (instance->adapter_type == VENTURA_SERIES)
2756                                 io_request->RaidContext.raid_context_g35.span_arm
2757                                         = io_info.span_arm;
2758                         else
2759                                 io_request->RaidContext.raid_context.span_arm
2760                                         = io_info.span_arm;
2761
2762                 } else
2763                         scp->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2764
2765                 if (instance->adapter_type == VENTURA_SERIES)
2766                         cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2767                 else
2768                         cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2769
2770                 if ((raidLUN[0] == 1) &&
2771                         (local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2772                         instance->dev_handle = !(instance->dev_handle);
2773                         io_info.devHandle =
2774                                 local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2775                 }
2776
2777                 cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2778                 io_request->DevHandle = io_info.devHandle;
2779                 cmd->pd_interface = io_info.pd_interface;
2780                 /* populate the LUN field */
2781                 memcpy(io_request->LUN, raidLUN, 8);
2782         } else {
2783                 io_request->RaidContext.raid_context.timeout_value =
2784                         cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2785                 cmd->request_desc->SCSIIO.RequestFlags =
2786                         (MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2787                          << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2788                 if (instance->adapter_type == INVADER_SERIES) {
2789                         if (io_info.do_fp_rlbypass ||
2790                         (io_request->RaidContext.raid_context.reg_lock_flags
2791                                         == REGION_TYPE_UNUSED))
2792                                 cmd->request_desc->SCSIIO.RequestFlags =
2793                                         (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2794                                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2795                         io_request->RaidContext.raid_context.type
2796                                 = MPI2_TYPE_CUDA;
2797                         io_request->RaidContext.raid_context.reg_lock_flags |=
2798                                 (MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2799                                  MR_RL_FLAGS_SEQ_NUM_ENABLE);
2800                         io_request->RaidContext.raid_context.nseg = 0x1;
2801                 } else if (instance->adapter_type == VENTURA_SERIES) {
2802                         io_request->RaidContext.raid_context_g35.routing_flags |=
2803                                         (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2804                         io_request->RaidContext.raid_context_g35.nseg_type |=
2805                                         (1 << RAID_CONTEXT_NSEG_SHIFT);
2806                         io_request->RaidContext.raid_context_g35.nseg_type |=
2807                                         (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2808                 }
2809                 io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2810                 io_request->DevHandle = cpu_to_le16(device_id);
2811
2812         } /* Not FP */
2813 }
2814
2815 /**
2816  * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
2817  * @instance:           Adapter soft state
2818  * @scp:                SCSI command
2819  * @cmd:                Command to be prepared
2820  *
2821  * Prepares the io_request frame for non-rw io cmds for vd.
2822  */
2823 static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2824                           struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2825 {
2826         u32 device_id;
2827         struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2828         u16 ld;
2829         struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2830         struct fusion_context *fusion = instance->ctrl_context;
2831         u8                          span, physArm;
2832         __le16                      devHandle;
2833         u32                         arRef, pd;
2834         struct MR_LD_RAID                  *raid;
2835         struct RAID_CONTEXT                *pRAID_Context;
2836         u8 fp_possible = 1;
2837
2838         io_request = cmd->io_request;
2839         device_id = MEGASAS_DEV_INDEX(scmd);
2840         local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2841         io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
2842         /* get RAID_Context pointer */
2843         pRAID_Context = &io_request->RaidContext.raid_context;
2844         /* Check with FW team */
2845         pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2846         pRAID_Context->reg_lock_row_lba    = 0;
2847         pRAID_Context->reg_lock_length    = 0;
2848
2849         if (fusion->fast_path_io && (
2850                 device_id < instance->fw_supported_vd_count)) {
2851
2852                 ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2853                 if (ld >= instance->fw_supported_vd_count)
2854                         fp_possible = 0;
2855                 else {
2856                         raid = MR_LdRaidGet(ld, local_map_ptr);
2857                         if (!(raid->capability.fpNonRWCapable))
2858                                 fp_possible = 0;
2859                 }
2860         } else
2861                 fp_possible = 0;
2862
2863         if (!fp_possible) {
2864                 io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2865                 io_request->DevHandle = cpu_to_le16(device_id);
2866                 io_request->LUN[1] = scmd->device->lun;
2867                 pRAID_Context->timeout_value =
2868                         cpu_to_le16 (scmd->request->timeout / HZ);
2869                 cmd->request_desc->SCSIIO.RequestFlags =
2870                         (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
2871                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2872         } else {
2873
2874                 /* set RAID context values */
2875                 pRAID_Context->config_seq_num = raid->seqNum;
2876                 if (instance->adapter_type != VENTURA_SERIES)
2877                         pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
2878                 pRAID_Context->timeout_value =
2879                         cpu_to_le16(raid->fpIoTimeoutForLd);
2880
2881                 /* get the DevHandle for the PD (since this is
2882                    fpNonRWCapable, this is a single disk RAID0) */
2883                 span = physArm = 0;
2884                 arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);
2885                 pd = MR_ArPdGet(arRef, physArm, local_map_ptr);
2886                 devHandle = MR_PdDevHandleGet(pd, local_map_ptr);
2887
2888                 /* build request descriptor */
2889                 cmd->request_desc->SCSIIO.RequestFlags =
2890                         (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
2891                         MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2892                 cmd->request_desc->SCSIIO.DevHandle = devHandle;
2893
2894                 /* populate the LUN field */
2895                 memcpy(io_request->LUN, raid->LUN, 8);
2896
2897                 /* build the raidScsiIO structure */
2898                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2899                 io_request->DevHandle = devHandle;
2900         }
2901 }
2902
2903 /**
2904  * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
2905  * @instance:           Adapter soft state
2906  * @scp:                SCSI command
2907  * @cmd:                Command to be prepared
2908  * @fp_possible:        parameter to detect fast path or firmware path io.
2909  *
2910  * Prepares the io_request frame for rw/non-rw io cmds for syspds
2911  */
2912 static void
2913 megasas_build_syspd_fusion(struct megasas_instance *instance,
2914         struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
2915         bool fp_possible)
2916 {
2917         u32 device_id;
2918         struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2919         u16 pd_index = 0;
2920         u16 os_timeout_value;
2921         u16 timeout_limit;
2922         struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2923         struct RAID_CONTEXT     *pRAID_Context;
2924         struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
2925         struct MR_PRIV_DEVICE *mr_device_priv_data;
2926         struct fusion_context *fusion = instance->ctrl_context;
2927         pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
2928
2929         device_id = MEGASAS_DEV_INDEX(scmd);
2930         pd_index = MEGASAS_PD_INDEX(scmd);
2931         os_timeout_value = scmd->request->timeout / HZ;
2932         mr_device_priv_data = scmd->device->hostdata;
2933         cmd->pd_interface = mr_device_priv_data->interface_type;
2934
2935         io_request = cmd->io_request;
2936         /* get RAID_Context pointer */
2937         pRAID_Context = &io_request->RaidContext.raid_context;
2938         pRAID_Context->reg_lock_flags = 0;
2939         pRAID_Context->reg_lock_row_lba = 0;
2940         pRAID_Context->reg_lock_length = 0;
2941         io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
2942         io_request->LUN[1] = scmd->device->lun;
2943         pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
2944                 << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
2945
2946         /* If FW supports PD sequence number */
2947         if (instance->use_seqnum_jbod_fp &&
2948                 instance->pd_list[pd_index].driveType == TYPE_DISK) {
2949                 /* TgtId must be incremented by 255 as jbod seq number is index
2950                  * below raid map
2951                  */
2952                  /* More than 256 PD/JBOD support for Ventura */
2953                 if (instance->support_morethan256jbod)
2954                         pRAID_Context->virtual_disk_tgt_id =
2955                                 pd_sync->seq[pd_index].pd_target_id;
2956                 else
2957                         pRAID_Context->virtual_disk_tgt_id =
2958                                 cpu_to_le16(device_id + (MAX_PHYSICAL_DEVICES - 1));
2959                 pRAID_Context->config_seq_num = pd_sync->seq[pd_index].seqNum;
2960                 io_request->DevHandle = pd_sync->seq[pd_index].devHandle;
2961                 if (instance->adapter_type == VENTURA_SERIES) {
2962                         io_request->RaidContext.raid_context_g35.routing_flags |=
2963                                 (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2964                         io_request->RaidContext.raid_context_g35.nseg_type |=
2965                                                         (1 << RAID_CONTEXT_NSEG_SHIFT);
2966                         io_request->RaidContext.raid_context_g35.nseg_type |=
2967                                                         (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2968                 } else {
2969                         pRAID_Context->type = MPI2_TYPE_CUDA;
2970                         pRAID_Context->nseg = 0x1;
2971                         pRAID_Context->reg_lock_flags |=
2972                                 (MR_RL_FLAGS_SEQ_NUM_ENABLE|MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
2973                 }
2974         } else if (fusion->fast_path_io) {
2975                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2976                 pRAID_Context->config_seq_num = 0;
2977                 local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2978                 io_request->DevHandle =
2979                         local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
2980         } else {
2981                 /* Want to send all IO via FW path */
2982                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
2983                 pRAID_Context->config_seq_num = 0;
2984                 io_request->DevHandle = cpu_to_le16(0xFFFF);
2985         }
2986
2987         cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
2988         cmd->request_desc->SCSIIO.MSIxIndex =
2989                 instance->msix_vectors ?
2990                 (raw_smp_processor_id() % instance->msix_vectors) : 0;
2991
2992
2993         if (!fp_possible) {
2994                 /* system pd firmware path */
2995                 io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2996                 cmd->request_desc->SCSIIO.RequestFlags =
2997                         (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
2998                                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2999                 pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
3000                 pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3001         } else {
3002                 /* system pd Fast Path */
3003                 io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3004                 timeout_limit = (scmd->device->type == TYPE_DISK) ?
3005                                 255 : 0xFFFF;
3006                 pRAID_Context->timeout_value =
3007                         cpu_to_le16((os_timeout_value > timeout_limit) ?
3008                         timeout_limit : os_timeout_value);
3009                 if (instance->adapter_type >= INVADER_SERIES)
3010                         io_request->IoFlags |=
3011                                 cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
3012
3013                 cmd->request_desc->SCSIIO.RequestFlags =
3014                         (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3015                                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3016         }
3017 }
3018
3019 /**
3020  * megasas_build_io_fusion -    Prepares IOs to devices
3021  * @instance:           Adapter soft state
3022  * @scp:                SCSI command
3023  * @cmd:                Command to be prepared
3024  *
3025  * Invokes helper functions to prepare request frames
3026  * and sets flags appropriate for IO/Non-IO cmd
3027  */
3028 int
3029 megasas_build_io_fusion(struct megasas_instance *instance,
3030                         struct scsi_cmnd *scp,
3031                         struct megasas_cmd_fusion *cmd)
3032 {
3033         int sge_count;
3034         u8  cmd_type;
3035         struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
3036         struct MR_PRIV_DEVICE *mr_device_priv_data;
3037         mr_device_priv_data = scp->device->hostdata;
3038
3039         /* Zero out some fields so they don't get reused */
3040         memset(io_request->LUN, 0x0, 8);
3041         io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
3042         io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
3043         io_request->EEDPFlags = 0;
3044         io_request->Control = 0;
3045         io_request->EEDPBlockSize = 0;
3046         io_request->ChainOffset = 0;
3047         io_request->RaidContext.raid_context.raid_flags = 0;
3048         io_request->RaidContext.raid_context.type = 0;
3049         io_request->RaidContext.raid_context.nseg = 0;
3050
3051         memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
3052         /*
3053          * Just the CDB length,rest of the Flags are zero
3054          * This will be modified for FP in build_ldio_fusion
3055          */
3056         io_request->IoFlags = cpu_to_le16(scp->cmd_len);
3057
3058         switch (cmd_type = megasas_cmd_type(scp)) {
3059         case READ_WRITE_LDIO:
3060                 megasas_build_ldio_fusion(instance, scp, cmd);
3061                 break;
3062         case NON_READ_WRITE_LDIO:
3063                 megasas_build_ld_nonrw_fusion(instance, scp, cmd);
3064                 break;
3065         case READ_WRITE_SYSPDIO:
3066                 megasas_build_syspd_fusion(instance, scp, cmd, true);
3067                 break;
3068         case NON_READ_WRITE_SYSPDIO:
3069                 if (instance->secure_jbod_support ||
3070                     mr_device_priv_data->is_tm_capable)
3071                         megasas_build_syspd_fusion(instance, scp, cmd, false);
3072                 else
3073                         megasas_build_syspd_fusion(instance, scp, cmd, true);
3074                 break;
3075         default:
3076                 break;
3077         }
3078
3079         /*
3080          * Construct SGL
3081          */
3082
3083         sge_count = megasas_make_sgl(instance, scp, cmd);
3084
3085         if (sge_count > instance->max_num_sge || (sge_count < 0)) {
3086                 dev_err(&instance->pdev->dev,
3087                         "%s %d sge_count (%d) is out of range. Range is:  0-%d\n",
3088                         __func__, __LINE__, sge_count, instance->max_num_sge);
3089                 return 1;
3090         }
3091
3092         if (instance->adapter_type == VENTURA_SERIES) {
3093                 set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);
3094                 cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
3095                 cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
3096         } else {
3097                 /* numSGE store lower 8 bit of sge_count.
3098                  * numSGEExt store higher 8 bit of sge_count
3099                  */
3100                 io_request->RaidContext.raid_context.num_sge = sge_count;
3101                 io_request->RaidContext.raid_context.num_sge_ext =
3102                         (u8)(sge_count >> 8);
3103         }
3104
3105         io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
3106
3107         if (scp->sc_data_direction == PCI_DMA_TODEVICE)
3108                 io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
3109         else if (scp->sc_data_direction == PCI_DMA_FROMDEVICE)
3110                 io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
3111
3112         io_request->SGLOffset0 =
3113                 offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
3114
3115         io_request->SenseBufferLowAddress =
3116                 cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));
3117         io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
3118
3119         cmd->scmd = scp;
3120         scp->SCp.ptr = (char *)cmd;
3121
3122         return 0;
3123 }
3124
3125 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3126 megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
3127 {
3128         u8 *p;
3129         struct fusion_context *fusion;
3130
3131         fusion = instance->ctrl_context;
3132         p = fusion->req_frames_desc +
3133                 sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
3134
3135         return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
3136 }
3137
3138
3139 /* megasas_prepate_secondRaid1_IO
3140  *  It prepares the raid 1 second IO
3141  */
3142 void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
3143                             struct megasas_cmd_fusion *cmd,
3144                             struct megasas_cmd_fusion *r1_cmd)
3145 {
3146         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
3147         struct fusion_context *fusion;
3148         fusion = instance->ctrl_context;
3149         req_desc = cmd->request_desc;
3150         /* copy the io request frame as well as 8 SGEs data for r1 command*/
3151         memcpy(r1_cmd->io_request, cmd->io_request,
3152                (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
3153         memcpy(&r1_cmd->io_request->SGL, &cmd->io_request->SGL,
3154                (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
3155         /*sense buffer is different for r1 command*/
3156         r1_cmd->io_request->SenseBufferLowAddress =
3157                         cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));
3158         r1_cmd->scmd = cmd->scmd;
3159         req_desc2 = megasas_get_request_descriptor(instance,
3160                                                    (r1_cmd->index - 1));
3161         req_desc2->Words = 0;
3162         r1_cmd->request_desc = req_desc2;
3163         req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
3164         req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
3165         r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
3166         r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
3167         r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
3168         cmd->io_request->RaidContext.raid_context_g35.smid.peer_smid =
3169                         cpu_to_le16(r1_cmd->index);
3170         r1_cmd->io_request->RaidContext.raid_context_g35.smid.peer_smid =
3171                         cpu_to_le16(cmd->index);
3172         /*MSIxIndex of both commands request descriptors should be same*/
3173         r1_cmd->request_desc->SCSIIO.MSIxIndex =
3174                         cmd->request_desc->SCSIIO.MSIxIndex;
3175         /*span arm is different for r1 cmd*/
3176         r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
3177                         cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
3178 }
3179
3180 /**
3181  * megasas_build_and_issue_cmd_fusion -Main routine for building and
3182  *                                     issuing non IOCTL cmd
3183  * @instance:                   Adapter soft state
3184  * @scmd:                       pointer to scsi cmd from OS
3185  */
3186 static u32
3187 megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
3188                                    struct scsi_cmnd *scmd)
3189 {
3190         struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
3191         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3192         u32 index;
3193         struct fusion_context *fusion;
3194
3195         fusion = instance->ctrl_context;
3196
3197         if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&
3198                 instance->ldio_threshold &&
3199                 (atomic_inc_return(&instance->ldio_outstanding) >
3200                 instance->ldio_threshold)) {
3201                 atomic_dec(&instance->ldio_outstanding);
3202                 return SCSI_MLQUEUE_DEVICE_BUSY;
3203         }
3204
3205         if (atomic_inc_return(&instance->fw_outstanding) >
3206                         instance->host->can_queue) {
3207                 atomic_dec(&instance->fw_outstanding);
3208                 return SCSI_MLQUEUE_HOST_BUSY;
3209         }
3210
3211         cmd = megasas_get_cmd_fusion(instance, scmd->request->tag);
3212
3213         if (!cmd) {
3214                 atomic_dec(&instance->fw_outstanding);
3215                 return SCSI_MLQUEUE_HOST_BUSY;
3216         }
3217
3218         index = cmd->index;
3219
3220         req_desc = megasas_get_request_descriptor(instance, index-1);
3221
3222         req_desc->Words = 0;
3223         cmd->request_desc = req_desc;
3224
3225         if (megasas_build_io_fusion(instance, scmd, cmd)) {
3226                 megasas_return_cmd_fusion(instance, cmd);
3227                 dev_err(&instance->pdev->dev, "Error building command\n");
3228                 cmd->request_desc = NULL;
3229                 atomic_dec(&instance->fw_outstanding);
3230                 return SCSI_MLQUEUE_HOST_BUSY;
3231         }
3232
3233         req_desc = cmd->request_desc;
3234         req_desc->SCSIIO.SMID = cpu_to_le16(index);
3235
3236         if (cmd->io_request->ChainOffset != 0 &&
3237             cmd->io_request->ChainOffset != 0xF)
3238                 dev_err(&instance->pdev->dev, "The chain offset value is not "
3239                        "correct : %x\n", cmd->io_request->ChainOffset);
3240         /*
3241          *      if it is raid 1/10 fp write capable.
3242          *      try to get second command from pool and construct it.
3243          *      From FW, it has confirmed that lba values of two PDs
3244          *      corresponds to single R1/10 LD are always same
3245          *
3246          */
3247         /*      driver side count always should be less than max_fw_cmds
3248          *      to get new command
3249          */
3250         if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
3251                 r1_cmd = megasas_get_cmd_fusion(instance,
3252                                 (scmd->request->tag + instance->max_fw_cmds));
3253                 megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
3254         }
3255
3256
3257         /*
3258          * Issue the command to the FW
3259          */
3260
3261         megasas_fire_cmd_fusion(instance, req_desc);
3262
3263         if (r1_cmd)
3264                 megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);
3265
3266
3267         return 0;
3268 }
3269
3270 /**
3271  * megasas_complete_r1_command -
3272  * completes R1 FP write commands which has valid peer smid
3273  * @instance:                   Adapter soft state
3274  * @cmd_fusion:                 MPT command frame
3275  *
3276  */
3277 static inline void
3278 megasas_complete_r1_command(struct megasas_instance *instance,
3279                             struct megasas_cmd_fusion *cmd)
3280 {
3281         u8 *sense, status, ex_status;
3282         u32 data_length;
3283         u16 peer_smid;
3284         struct fusion_context *fusion;
3285         struct megasas_cmd_fusion *r1_cmd = NULL;
3286         struct scsi_cmnd *scmd_local = NULL;
3287         struct RAID_CONTEXT_G35 *rctx_g35;
3288
3289         rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
3290         fusion = instance->ctrl_context;
3291         peer_smid = le16_to_cpu(rctx_g35->smid.peer_smid);
3292
3293         r1_cmd = fusion->cmd_list[peer_smid - 1];
3294         scmd_local = cmd->scmd;
3295         status = rctx_g35->status;
3296         ex_status = rctx_g35->ex_status;
3297         data_length = cmd->io_request->DataLength;
3298         sense = cmd->sense;
3299
3300         cmd->cmd_completed = true;
3301
3302         /* Check if peer command is completed or not*/
3303         if (r1_cmd->cmd_completed) {
3304                 rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
3305                 if (rctx_g35->status != MFI_STAT_OK) {
3306                         status = rctx_g35->status;
3307                         ex_status = rctx_g35->ex_status;
3308                         data_length = r1_cmd->io_request->DataLength;
3309                         sense = r1_cmd->sense;
3310                 }
3311
3312                 megasas_return_cmd_fusion(instance, r1_cmd);
3313                 map_cmd_status(fusion, scmd_local, status, ex_status,
3314                                le32_to_cpu(data_length), sense);
3315                 if (instance->ldio_threshold &&
3316                     megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
3317                         atomic_dec(&instance->ldio_outstanding);
3318                 scmd_local->SCp.ptr = NULL;
3319                 megasas_return_cmd_fusion(instance, cmd);
3320                 scsi_dma_unmap(scmd_local);
3321                 scmd_local->scsi_done(scmd_local);
3322         }
3323 }
3324
3325 /**
3326  * complete_cmd_fusion -        Completes command
3327  * @instance:                   Adapter soft state
3328  * Completes all commands that is in reply descriptor queue
3329  */
3330 int
3331 complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex)
3332 {
3333         union MPI2_REPLY_DESCRIPTORS_UNION *desc;
3334         struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
3335         struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
3336         struct fusion_context *fusion;
3337         struct megasas_cmd *cmd_mfi;
3338         struct megasas_cmd_fusion *cmd_fusion;
3339         u16 smid, num_completed;
3340         u8 reply_descript_type, *sense, status, extStatus;
3341         u32 device_id, data_length;
3342         union desc_value d_val;
3343         struct LD_LOAD_BALANCE_INFO *lbinfo;
3344         int threshold_reply_count = 0;
3345         struct scsi_cmnd *scmd_local = NULL;
3346         struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
3347         struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
3348
3349         fusion = instance->ctrl_context;
3350
3351         if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3352                 return IRQ_HANDLED;
3353
3354         desc = fusion->reply_frames_desc[MSIxIndex] +
3355                                 fusion->last_reply_idx[MSIxIndex];
3356
3357         reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3358
3359         d_val.word = desc->Words;
3360
3361         reply_descript_type = reply_desc->ReplyFlags &
3362                 MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3363
3364         if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3365                 return IRQ_NONE;
3366
3367         num_completed = 0;
3368
3369         while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3370                d_val.u.high != cpu_to_le32(UINT_MAX)) {
3371
3372                 smid = le16_to_cpu(reply_desc->SMID);
3373                 cmd_fusion = fusion->cmd_list[smid - 1];
3374                 scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3375                                                 cmd_fusion->io_request;
3376
3377                 scmd_local = cmd_fusion->scmd;
3378                 status = scsi_io_req->RaidContext.raid_context.status;
3379                 extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3380                 sense = cmd_fusion->sense;
3381                 data_length = scsi_io_req->DataLength;
3382
3383                 switch (scsi_io_req->Function) {
3384                 case MPI2_FUNCTION_SCSI_TASK_MGMT:
3385                         mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3386                                                 cmd_fusion->io_request;
3387                         mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3388                                                 &mr_tm_req->TmRequest;
3389                         dev_dbg(&instance->pdev->dev, "TM completion:"
3390                                 "type: 0x%x TaskMID: 0x%x\n",
3391                                 mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3392                         complete(&cmd_fusion->done);
3393                         break;
3394                 case MPI2_FUNCTION_SCSI_IO_REQUEST:  /*Fast Path IO.*/
3395                         /* Update load balancing info */
3396                         if (fusion->load_balance_info &&
3397                             (cmd_fusion->scmd->SCp.Status &
3398                             MEGASAS_LOAD_BALANCE_FLAG)) {
3399                                 device_id = MEGASAS_DEV_INDEX(scmd_local);
3400                                 lbinfo = &fusion->load_balance_info[device_id];
3401                                 atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3402                                 cmd_fusion->scmd->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
3403                         }
3404                         //Fall thru and complete IO
3405                 case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */
3406                         atomic_dec(&instance->fw_outstanding);
3407                         if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3408                                 map_cmd_status(fusion, scmd_local, status,
3409                                                extStatus, le32_to_cpu(data_length),
3410                                                sense);
3411                                 if (instance->ldio_threshold &&
3412                                     (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))
3413                                         atomic_dec(&instance->ldio_outstanding);
3414                                 scmd_local->SCp.ptr = NULL;
3415                                 megasas_return_cmd_fusion(instance, cmd_fusion);
3416                                 scsi_dma_unmap(scmd_local);
3417                                 scmd_local->scsi_done(scmd_local);
3418                         } else  /* Optimal VD - R1 FP command completion. */
3419                                 megasas_complete_r1_command(instance, cmd_fusion);
3420                         break;
3421                 case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */
3422                         cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3423                         /* Poll mode. Dummy free.
3424                          * In case of Interrupt mode, caller has reverse check.
3425                          */
3426                         if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3427                                 cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3428                                 megasas_return_cmd(instance, cmd_mfi);
3429                         } else
3430                                 megasas_complete_cmd(instance, cmd_mfi, DID_OK);
3431                         break;
3432                 }
3433
3434                 fusion->last_reply_idx[MSIxIndex]++;
3435                 if (fusion->last_reply_idx[MSIxIndex] >=
3436                     fusion->reply_q_depth)
3437                         fusion->last_reply_idx[MSIxIndex] = 0;
3438
3439                 desc->Words = cpu_to_le64(ULLONG_MAX);
3440                 num_completed++;
3441                 threshold_reply_count++;
3442
3443                 /* Get the next reply descriptor */
3444                 if (!fusion->last_reply_idx[MSIxIndex])
3445                         desc = fusion->reply_frames_desc[MSIxIndex];
3446                 else
3447                         desc++;
3448
3449                 reply_desc =
3450                   (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3451
3452                 d_val.word = desc->Words;
3453
3454                 reply_descript_type = reply_desc->ReplyFlags &
3455                         MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3456
3457                 if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3458                         break;
3459                 /*
3460                  * Write to reply post host index register after completing threshold
3461                  * number of reply counts and still there are more replies in reply queue
3462                  * pending to be completed
3463                  */
3464                 if (threshold_reply_count >= THRESHOLD_REPLY_COUNT) {
3465                         if (instance->msix_combined)
3466                                 writel(((MSIxIndex & 0x7) << 24) |
3467                                         fusion->last_reply_idx[MSIxIndex],
3468                                         instance->reply_post_host_index_addr[MSIxIndex/8]);
3469                         else
3470                                 writel((MSIxIndex << 24) |
3471                                         fusion->last_reply_idx[MSIxIndex],
3472                                         instance->reply_post_host_index_addr[0]);
3473                         threshold_reply_count = 0;
3474                 }
3475         }
3476
3477         if (!num_completed)
3478                 return IRQ_NONE;
3479
3480         wmb();
3481         if (instance->msix_combined)
3482                 writel(((MSIxIndex & 0x7) << 24) |
3483                         fusion->last_reply_idx[MSIxIndex],
3484                         instance->reply_post_host_index_addr[MSIxIndex/8]);
3485         else
3486                 writel((MSIxIndex << 24) |
3487                         fusion->last_reply_idx[MSIxIndex],
3488                         instance->reply_post_host_index_addr[0]);
3489         megasas_check_and_restore_queue_depth(instance);
3490         return IRQ_HANDLED;
3491 }
3492
3493 /**
3494  * megasas_sync_irqs -  Synchronizes all IRQs owned by adapter
3495  * @instance:                   Adapter soft state
3496  */
3497 void megasas_sync_irqs(unsigned long instance_addr)
3498 {
3499         u32 count, i;
3500         struct megasas_instance *instance =
3501                 (struct megasas_instance *)instance_addr;
3502
3503         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3504
3505         for (i = 0; i < count; i++)
3506                 synchronize_irq(pci_irq_vector(instance->pdev, i));
3507 }
3508
3509 /**
3510  * megasas_complete_cmd_dpc_fusion -    Completes command
3511  * @instance:                   Adapter soft state
3512  *
3513  * Tasklet to complete cmds
3514  */
3515 void
3516 megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
3517 {
3518         struct megasas_instance *instance =
3519                 (struct megasas_instance *)instance_addr;
3520         unsigned long flags;
3521         u32 count, MSIxIndex;
3522
3523         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3524
3525         /* If we have already declared adapter dead, donot complete cmds */
3526         spin_lock_irqsave(&instance->hba_lock, flags);
3527         if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
3528                 spin_unlock_irqrestore(&instance->hba_lock, flags);
3529                 return;
3530         }
3531         spin_unlock_irqrestore(&instance->hba_lock, flags);
3532
3533         for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++)
3534                 complete_cmd_fusion(instance, MSIxIndex);
3535 }
3536
3537 /**
3538  * megasas_isr_fusion - isr entry point
3539  */
3540 irqreturn_t megasas_isr_fusion(int irq, void *devp)
3541 {
3542         struct megasas_irq_context *irq_context = devp;
3543         struct megasas_instance *instance = irq_context->instance;
3544         u32 mfiStatus, fw_state, dma_state;
3545
3546         if (instance->mask_interrupts)
3547                 return IRQ_NONE;
3548
3549         if (!instance->msix_vectors) {
3550                 mfiStatus = instance->instancet->clear_intr(instance->reg_set);
3551                 if (!mfiStatus)
3552                         return IRQ_NONE;
3553         }
3554
3555         /* If we are resetting, bail */
3556         if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {
3557                 instance->instancet->clear_intr(instance->reg_set);
3558                 return IRQ_HANDLED;
3559         }
3560
3561         if (!complete_cmd_fusion(instance, irq_context->MSIxIndex)) {
3562                 instance->instancet->clear_intr(instance->reg_set);
3563                 /* If we didn't complete any commands, check for FW fault */
3564                 fw_state = instance->instancet->read_fw_status_reg(
3565                         instance->reg_set) & MFI_STATE_MASK;
3566                 dma_state = instance->instancet->read_fw_status_reg
3567                         (instance->reg_set) & MFI_STATE_DMADONE;
3568                 if (instance->crash_dump_drv_support &&
3569                         instance->crash_dump_app_support) {
3570                         /* Start collecting crash, if DMA bit is done */
3571                         if ((fw_state == MFI_STATE_FAULT) && dma_state)
3572                                 schedule_work(&instance->crash_init);
3573                         else if (fw_state == MFI_STATE_FAULT) {
3574                                 if (instance->unload == 0)
3575                                         schedule_work(&instance->work_init);
3576                         }
3577                 } else if (fw_state == MFI_STATE_FAULT) {
3578                         dev_warn(&instance->pdev->dev, "Iop2SysDoorbellInt"
3579                                "for scsi%d\n", instance->host->host_no);
3580                         if (instance->unload == 0)
3581                                 schedule_work(&instance->work_init);
3582                 }
3583         }
3584
3585         return IRQ_HANDLED;
3586 }
3587
3588 /**
3589  * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
3590  * @instance:                   Adapter soft state
3591  * mfi_cmd:                     megasas_cmd pointer
3592  *
3593  */
3594 void
3595 build_mpt_mfi_pass_thru(struct megasas_instance *instance,
3596                         struct megasas_cmd *mfi_cmd)
3597 {
3598         struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;
3599         struct MPI2_RAID_SCSI_IO_REQUEST *io_req;
3600         struct megasas_cmd_fusion *cmd;
3601         struct fusion_context *fusion;
3602         struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;
3603
3604         fusion = instance->ctrl_context;
3605
3606         cmd = megasas_get_cmd_fusion(instance,
3607                         instance->max_scsi_cmds + mfi_cmd->index);
3608
3609         /*  Save the smid. To be used for returning the cmd */
3610         mfi_cmd->context.smid = cmd->index;
3611
3612         /*
3613          * For cmds where the flag is set, store the flag and check
3614          * on completion. For cmds with this flag, don't call
3615          * megasas_complete_cmd
3616          */
3617
3618         if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))
3619                 mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;
3620
3621         io_req = cmd->io_request;
3622
3623         if (instance->adapter_type >= INVADER_SERIES) {
3624                 struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =
3625                         (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;
3626                 sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
3627                 sgl_ptr_end->Flags = 0;
3628         }
3629
3630         mpi25_ieee_chain =
3631           (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;
3632
3633         io_req->Function    = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
3634         io_req->SGLOffset0  = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,
3635                                        SGL) / 4;
3636         io_req->ChainOffset = fusion->chain_offset_mfi_pthru;
3637
3638         mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);
3639
3640         mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
3641                 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
3642
3643         mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);
3644 }
3645
3646 /**
3647  * build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd
3648  * @instance:                   Adapter soft state
3649  * @cmd:                        mfi cmd to build
3650  *
3651  */
3652 union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3653 build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
3654 {
3655         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;
3656         u16 index;
3657
3658         build_mpt_mfi_pass_thru(instance, cmd);
3659         index = cmd->context.smid;
3660
3661         req_desc = megasas_get_request_descriptor(instance, index - 1);
3662
3663         req_desc->Words = 0;
3664         req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3665                                          MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3666
3667         req_desc->SCSIIO.SMID = cpu_to_le16(index);
3668
3669         return req_desc;
3670 }
3671
3672 /**
3673  * megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd
3674  * @instance:                   Adapter soft state
3675  * @cmd:                        mfi cmd pointer
3676  *
3677  */
3678 void
3679 megasas_issue_dcmd_fusion(struct megasas_instance *instance,
3680                           struct megasas_cmd *cmd)
3681 {
3682         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3683
3684         req_desc = build_mpt_cmd(instance, cmd);
3685
3686         megasas_fire_cmd_fusion(instance, req_desc);
3687         return;
3688 }
3689
3690 /**
3691  * megasas_release_fusion -     Reverses the FW initialization
3692  * @instance:                   Adapter soft state
3693  */
3694 void
3695 megasas_release_fusion(struct megasas_instance *instance)
3696 {
3697         megasas_free_ioc_init_cmd(instance);
3698         megasas_free_cmds(instance);
3699         megasas_free_cmds_fusion(instance);
3700
3701         iounmap(instance->reg_set);
3702
3703         pci_release_selected_regions(instance->pdev, 1<<instance->bar);
3704 }
3705
3706 /**
3707  * megasas_read_fw_status_reg_fusion - returns the current FW status value
3708  * @regs:                       MFI register set
3709  */
3710 static u32
3711 megasas_read_fw_status_reg_fusion(struct megasas_register_set __iomem *regs)
3712 {
3713         return readl(&(regs)->outbound_scratch_pad);
3714 }
3715
3716 /**
3717  * megasas_alloc_host_crash_buffer -    Host buffers for Crash dump collection from Firmware
3718  * @instance:                           Controller's soft instance
3719  * return:                              Number of allocated host crash buffers
3720  */
3721 static void
3722 megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
3723 {
3724         unsigned int i;
3725
3726         for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
3727                 instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);
3728                 if (!instance->crash_buf[i]) {
3729                         dev_info(&instance->pdev->dev, "Firmware crash dump "
3730                                 "memory allocation failed at index %d\n", i);
3731                         break;
3732                 }
3733         }
3734         instance->drv_buf_alloc = i;
3735 }
3736
3737 /**
3738  * megasas_free_host_crash_buffer -     Host buffers for Crash dump collection from Firmware
3739  * @instance:                           Controller's soft instance
3740  */
3741 void
3742 megasas_free_host_crash_buffer(struct megasas_instance *instance)
3743 {
3744         unsigned int i;
3745         for (i = 0; i < instance->drv_buf_alloc; i++) {
3746                 if (instance->crash_buf[i])
3747                         vfree(instance->crash_buf[i]);
3748         }
3749         instance->drv_buf_index = 0;
3750         instance->drv_buf_alloc = 0;
3751         instance->fw_crash_state = UNAVAILABLE;
3752         instance->fw_crash_buffer_size = 0;
3753 }
3754
3755 /**
3756  * megasas_adp_reset_fusion -   For controller reset
3757  * @regs:                               MFI register set
3758  */
3759 static int
3760 megasas_adp_reset_fusion(struct megasas_instance *instance,
3761                          struct megasas_register_set __iomem *regs)
3762 {
3763         u32 host_diag, abs_state, retry;
3764
3765         /* Now try to reset the chip */
3766         writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3767         writel(MPI2_WRSEQ_1ST_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3768         writel(MPI2_WRSEQ_2ND_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3769         writel(MPI2_WRSEQ_3RD_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3770         writel(MPI2_WRSEQ_4TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3771         writel(MPI2_WRSEQ_5TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3772         writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
3773
3774         /* Check that the diag write enable (DRWE) bit is on */
3775         host_diag = readl(&instance->reg_set->fusion_host_diag);
3776         retry = 0;
3777         while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {
3778                 msleep(100);
3779                 host_diag = readl(&instance->reg_set->fusion_host_diag);
3780                 if (retry++ == 100) {
3781                         dev_warn(&instance->pdev->dev,
3782                                 "Host diag unlock failed from %s %d\n",
3783                                 __func__, __LINE__);
3784                         break;
3785                 }
3786         }
3787         if (!(host_diag & HOST_DIAG_WRITE_ENABLE))
3788                 return -1;
3789
3790         /* Send chip reset command */
3791         writel(host_diag | HOST_DIAG_RESET_ADAPTER,
3792                 &instance->reg_set->fusion_host_diag);
3793         msleep(3000);
3794
3795         /* Make sure reset adapter bit is cleared */
3796         host_diag = readl(&instance->reg_set->fusion_host_diag);
3797         retry = 0;
3798         while (host_diag & HOST_DIAG_RESET_ADAPTER) {
3799                 msleep(100);
3800                 host_diag = readl(&instance->reg_set->fusion_host_diag);
3801                 if (retry++ == 1000) {
3802                         dev_warn(&instance->pdev->dev,
3803                                 "Diag reset adapter never cleared %s %d\n",
3804                                 __func__, __LINE__);
3805                         break;
3806                 }
3807         }
3808         if (host_diag & HOST_DIAG_RESET_ADAPTER)
3809                 return -1;
3810
3811         abs_state = instance->instancet->read_fw_status_reg(instance->reg_set)
3812                         & MFI_STATE_MASK;
3813         retry = 0;
3814
3815         while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {
3816                 msleep(100);
3817                 abs_state = instance->instancet->
3818                         read_fw_status_reg(instance->reg_set) & MFI_STATE_MASK;
3819         }
3820         if (abs_state <= MFI_STATE_FW_INIT) {
3821                 dev_warn(&instance->pdev->dev,
3822                         "fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",
3823                         abs_state, __func__, __LINE__);
3824                 return -1;
3825         }
3826
3827         return 0;
3828 }
3829
3830 /**
3831  * megasas_check_reset_fusion - For controller reset check
3832  * @regs:                               MFI register set
3833  */
3834 static int
3835 megasas_check_reset_fusion(struct megasas_instance *instance,
3836                            struct megasas_register_set __iomem *regs)
3837 {
3838         return 0;
3839 }
3840
3841 /* This function waits for outstanding commands on fusion to complete */
3842 int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
3843                                         int reason, int *convert)
3844 {
3845         int i, outstanding, retval = 0, hb_seconds_missed = 0;
3846         u32 fw_state;
3847
3848         for (i = 0; i < resetwaittime; i++) {
3849                 /* Check if firmware is in fault state */
3850                 fw_state = instance->instancet->read_fw_status_reg(
3851                         instance->reg_set) & MFI_STATE_MASK;
3852                 if (fw_state == MFI_STATE_FAULT) {
3853                         dev_warn(&instance->pdev->dev, "Found FW in FAULT state,"
3854                                " will reset adapter scsi%d.\n",
3855                                 instance->host->host_no);
3856                         megasas_complete_cmd_dpc_fusion((unsigned long)instance);
3857                         if (instance->requestorId && reason) {
3858                                 dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"
3859                                 " state while polling during"
3860                                 " I/O timeout handling for %d\n",
3861                                 instance->host->host_no);
3862                                 *convert = 1;
3863                         }
3864
3865                         retval = 1;
3866                         goto out;
3867                 }
3868
3869                 if (reason == MFI_IO_TIMEOUT_OCR) {
3870                         dev_info(&instance->pdev->dev,
3871                                 "MFI IO is timed out, initiating OCR\n");
3872                         megasas_complete_cmd_dpc_fusion((unsigned long)instance);
3873                         retval = 1;
3874                         goto out;
3875                 }
3876
3877                 /* If SR-IOV VF mode & heartbeat timeout, don't wait */
3878                 if (instance->requestorId && !reason) {
3879                         retval = 1;
3880                         goto out;
3881                 }
3882
3883                 /* If SR-IOV VF mode & I/O timeout, check for HB timeout */
3884                 if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {
3885                         if (instance->hb_host_mem->HB.fwCounter !=
3886                             instance->hb_host_mem->HB.driverCounter) {
3887                                 instance->hb_host_mem->HB.driverCounter =
3888                                         instance->hb_host_mem->HB.fwCounter;
3889                                 hb_seconds_missed = 0;
3890                         } else {
3891                                 hb_seconds_missed++;
3892                                 if (hb_seconds_missed ==
3893                                     (MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {
3894                                         dev_warn(&instance->pdev->dev, "SR-IOV:"
3895                                                " Heartbeat never completed "
3896                                                " while polling during I/O "
3897                                                " timeout handling for "
3898                                                "scsi%d.\n",
3899                                                instance->host->host_no);
3900                                                *convert = 1;
3901                                                retval = 1;
3902                                                goto out;
3903                                 }
3904                         }
3905                 }
3906
3907                 megasas_complete_cmd_dpc_fusion((unsigned long)instance);
3908                 outstanding = atomic_read(&instance->fw_outstanding);
3909                 if (!outstanding)
3910                         goto out;
3911
3912                 if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {
3913                         dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
3914                                "commands to complete for scsi%d\n", i,
3915                                outstanding, instance->host->host_no);
3916                 }
3917                 msleep(1000);
3918         }
3919
3920         if (atomic_read(&instance->fw_outstanding)) {
3921                 dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
3922                        "will reset adapter scsi%d.\n",
3923                        instance->host->host_no);
3924                 *convert = 1;
3925                 retval = 1;
3926         }
3927 out:
3928         return retval;
3929 }
3930
3931 void  megasas_reset_reply_desc(struct megasas_instance *instance)
3932 {
3933         int i, j, count;
3934         struct fusion_context *fusion;
3935         union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
3936
3937         fusion = instance->ctrl_context;
3938         count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3939         for (i = 0 ; i < count ; i++) {
3940                 fusion->last_reply_idx[i] = 0;
3941                 reply_desc = fusion->reply_frames_desc[i];
3942                 for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)
3943                         reply_desc->Words = cpu_to_le64(ULLONG_MAX);
3944         }
3945 }
3946
3947 /*
3948  * megasas_refire_mgmt_cmd :    Re-fire management commands
3949  * @instance:                           Controller's soft instance
3950 */
3951 void megasas_refire_mgmt_cmd(struct megasas_instance *instance)
3952 {
3953         int j;
3954         struct megasas_cmd_fusion *cmd_fusion;
3955         struct fusion_context *fusion;
3956         struct megasas_cmd *cmd_mfi;
3957         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3958         u16 smid;
3959         bool refire_cmd = 0;
3960         u8 result;
3961         u32 opcode = 0;
3962
3963         fusion = instance->ctrl_context;
3964
3965         /* Re-fire management commands.
3966          * Do not traverse complet MPT frame pool. Start from max_scsi_cmds.
3967          */
3968         for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {
3969                 cmd_fusion = fusion->cmd_list[j];
3970                 cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3971                 smid = le16_to_cpu(cmd_mfi->context.smid);
3972                 result = REFIRE_CMD;
3973
3974                 if (!smid)
3975                         continue;
3976
3977                 req_desc = megasas_get_request_descriptor(instance, smid - 1);
3978
3979                 switch (cmd_mfi->frame->hdr.cmd) {
3980                 case MFI_CMD_DCMD:
3981                         opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode);
3982                          /* Do not refire shutdown command */
3983                         if (opcode == MR_DCMD_CTRL_SHUTDOWN) {
3984                                 cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;
3985                                 result = COMPLETE_CMD;
3986                                 break;
3987                         }
3988
3989                         refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) &&
3990                                       (opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&
3991                                       !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);
3992
3993                         if (!refire_cmd)
3994                                 result = RETURN_CMD;
3995
3996                         break;
3997                 case MFI_CMD_NVME:
3998                         if (!instance->support_nvme_passthru) {
3999                                 cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4000                                 result = COMPLETE_CMD;
4001                         }
4002
4003                         break;
4004                 default:
4005                         break;
4006                 }
4007
4008                 switch (result) {
4009                 case REFIRE_CMD:
4010                         megasas_fire_cmd_fusion(instance, req_desc);
4011                         break;
4012                 case RETURN_CMD:
4013                         megasas_return_cmd(instance, cmd_mfi);
4014                         break;
4015                 case COMPLETE_CMD:
4016                         megasas_complete_cmd(instance, cmd_mfi, DID_OK);
4017                         break;
4018                 }
4019         }
4020 }
4021
4022 /*
4023  * megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device
4024  * @instance: per adapter struct
4025  * @channel: the channel assigned by the OS
4026  * @id: the id assigned by the OS
4027  *
4028  * Returns SUCCESS if no IOs pending to SCSI device, else return FAILED
4029  */
4030
4031 static int megasas_track_scsiio(struct megasas_instance *instance,
4032                 int id, int channel)
4033 {
4034         int i, found = 0;
4035         struct megasas_cmd_fusion *cmd_fusion;
4036         struct fusion_context *fusion;
4037         fusion = instance->ctrl_context;
4038
4039         for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4040                 cmd_fusion = fusion->cmd_list[i];
4041                 if (cmd_fusion->scmd &&
4042                         (cmd_fusion->scmd->device->id == id &&
4043                         cmd_fusion->scmd->device->channel == channel)) {
4044                         dev_info(&instance->pdev->dev,
4045                                 "SCSI commands pending to target"
4046                                 "channel %d id %d \tSMID: 0x%x\n",
4047                                 channel, id, cmd_fusion->index);
4048                         scsi_print_command(cmd_fusion->scmd);
4049                         found = 1;
4050                         break;
4051                 }
4052         }
4053
4054         return found ? FAILED : SUCCESS;
4055 }
4056
4057 /**
4058  * megasas_tm_response_code - translation of device response code
4059  * @ioc: per adapter object
4060  * @mpi_reply: MPI reply returned by firmware
4061  *
4062  * Return nothing.
4063  */
4064 static void
4065 megasas_tm_response_code(struct megasas_instance *instance,
4066                 struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
4067 {
4068         char *desc;
4069
4070         switch (mpi_reply->ResponseCode) {
4071         case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
4072                 desc = "task management request completed";
4073                 break;
4074         case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
4075                 desc = "invalid frame";
4076                 break;
4077         case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
4078                 desc = "task management request not supported";
4079                 break;
4080         case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
4081                 desc = "task management request failed";
4082                 break;
4083         case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
4084                 desc = "task management request succeeded";
4085                 break;
4086         case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
4087                 desc = "invalid lun";
4088                 break;
4089         case 0xA:
4090                 desc = "overlapped tag attempted";
4091                 break;
4092         case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
4093                 desc = "task queued, however not sent to target";
4094                 break;
4095         default:
4096                 desc = "unknown";
4097                 break;
4098         }
4099         dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",
4100                 mpi_reply->ResponseCode, desc);
4101         dev_dbg(&instance->pdev->dev,
4102                 "TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"
4103                 " 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
4104                 mpi_reply->TerminationCount, mpi_reply->DevHandle,
4105                 mpi_reply->Function, mpi_reply->TaskType,
4106                 mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
4107 }
4108
4109 /**
4110  * megasas_issue_tm - main routine for sending tm requests
4111  * @instance: per adapter struct
4112  * @device_handle: device handle
4113  * @channel: the channel assigned by the OS
4114  * @id: the id assigned by the OS
4115  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
4116  * @smid_task: smid assigned to the task
4117  * @m_type: TM_MUTEX_ON or TM_MUTEX_OFF
4118  * Context: user
4119  *
4120  * MegaRaid use MPT interface for Task Magement request.
4121  * A generic API for sending task management requests to firmware.
4122  *
4123  * Return SUCCESS or FAILED.
4124  */
4125 static int
4126 megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
4127         uint channel, uint id, u16 smid_task, u8 type)
4128 {
4129         struct MR_TASK_MANAGE_REQUEST *mr_request;
4130         struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;
4131         unsigned long timeleft;
4132         struct megasas_cmd_fusion *cmd_fusion;
4133         struct megasas_cmd *cmd_mfi;
4134         union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4135         struct fusion_context *fusion = NULL;
4136         struct megasas_cmd_fusion *scsi_lookup;
4137         int rc;
4138         struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
4139
4140         fusion = instance->ctrl_context;
4141
4142         cmd_mfi = megasas_get_cmd(instance);
4143
4144         if (!cmd_mfi) {
4145                 dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4146                         __func__, __LINE__);
4147                 return -ENOMEM;
4148         }
4149
4150         cmd_fusion = megasas_get_cmd_fusion(instance,
4151                         instance->max_scsi_cmds + cmd_mfi->index);
4152
4153         /*  Save the smid. To be used for returning the cmd */
4154         cmd_mfi->context.smid = cmd_fusion->index;
4155
4156         req_desc = megasas_get_request_descriptor(instance,
4157                         (cmd_fusion->index - 1));
4158
4159         cmd_fusion->request_desc = req_desc;
4160         req_desc->Words = 0;
4161
4162         mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
4163         memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
4164         mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
4165         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4166         mpi_request->DevHandle = cpu_to_le16(device_handle);
4167         mpi_request->TaskType = type;
4168         mpi_request->TaskMID = cpu_to_le16(smid_task);
4169         mpi_request->LUN[1] = 0;
4170
4171
4172         req_desc = cmd_fusion->request_desc;
4173         req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);
4174         req_desc->HighPriority.RequestFlags =
4175                 (MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
4176                 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
4177         req_desc->HighPriority.MSIxIndex =  0;
4178         req_desc->HighPriority.LMID = 0;
4179         req_desc->HighPriority.Reserved1 = 0;
4180
4181         if (channel < MEGASAS_MAX_PD_CHANNELS)
4182                 mr_request->tmReqFlags.isTMForPD = 1;
4183         else
4184                 mr_request->tmReqFlags.isTMForLD = 1;
4185
4186         init_completion(&cmd_fusion->done);
4187         megasas_fire_cmd_fusion(instance, req_desc);
4188
4189         timeleft = wait_for_completion_timeout(&cmd_fusion->done, 50 * HZ);
4190
4191         if (!timeleft) {
4192                 dev_err(&instance->pdev->dev,
4193                         "task mgmt type 0x%x timed out\n", type);
4194                 cmd_mfi->flags |= DRV_DCMD_SKIP_REFIRE;
4195                 mutex_unlock(&instance->reset_mutex);
4196                 rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);
4197                 mutex_lock(&instance->reset_mutex);
4198                 return rc;
4199         }
4200
4201         mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;
4202         megasas_tm_response_code(instance, mpi_reply);
4203
4204         megasas_return_cmd(instance, cmd_mfi);
4205         rc = SUCCESS;
4206         switch (type) {
4207         case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4208                 scsi_lookup = fusion->cmd_list[smid_task - 1];
4209
4210                 if (scsi_lookup->scmd == NULL)
4211                         break;
4212                 else {
4213                         instance->instancet->disable_intr(instance);
4214                         megasas_sync_irqs((unsigned long)instance);
4215                         instance->instancet->enable_intr(instance);
4216                         if (scsi_lookup->scmd == NULL)
4217                                 break;
4218                 }
4219                 rc = FAILED;
4220                 break;
4221
4222         case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4223                 if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
4224                         break;
4225                 instance->instancet->disable_intr(instance);
4226                 megasas_sync_irqs((unsigned long)instance);
4227                 rc = megasas_track_scsiio(instance, id, channel);
4228                 instance->instancet->enable_intr(instance);
4229
4230                 break;
4231         case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
4232         case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
4233                 break;
4234         default:
4235                 rc = FAILED;
4236                 break;
4237         }
4238
4239         return rc;
4240
4241 }
4242
4243 /*
4244  * megasas_fusion_smid_lookup : Look for fusion command correpspodning to SCSI
4245  * @instance: per adapter struct
4246  *
4247  * Return Non Zero index, if SMID found in outstanding commands
4248  */
4249 static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)
4250 {
4251         int i, ret = 0;
4252         struct megasas_instance *instance;
4253         struct megasas_cmd_fusion *cmd_fusion;
4254         struct fusion_context *fusion;
4255
4256         instance = (struct megasas_instance *)scmd->device->host->hostdata;
4257
4258         fusion = instance->ctrl_context;
4259
4260         for (i = 0; i < instance->max_scsi_cmds; i++) {
4261                 cmd_fusion = fusion->cmd_list[i];
4262                 if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {
4263                         scmd_printk(KERN_NOTICE, scmd, "Abort request is for"
4264                                 " SMID: %d\n", cmd_fusion->index);
4265                         ret = cmd_fusion->index;
4266                         break;
4267                 }
4268         }
4269
4270         return ret;
4271 }
4272
4273 /*
4274 * megasas_get_tm_devhandle - Get devhandle for TM request
4275 * @sdev-                     OS provided scsi device
4276 *
4277 * Returns-                   devhandle/targetID of SCSI device
4278 */
4279 static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)
4280 {
4281         u16 pd_index = 0;
4282         u32 device_id;
4283         struct megasas_instance *instance;
4284         struct fusion_context *fusion;
4285         struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
4286         u16 devhandle = (u16)ULONG_MAX;
4287
4288         instance = (struct megasas_instance *)sdev->host->hostdata;
4289         fusion = instance->ctrl_context;
4290
4291         if (!MEGASAS_IS_LOGICAL(sdev)) {
4292                 if (instance->use_seqnum_jbod_fp) {
4293                         pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)
4294                                     + sdev->id;
4295                         pd_sync = (void *)fusion->pd_seq_sync
4296                                         [(instance->pd_seq_map_id - 1) & 1];
4297                         devhandle = pd_sync->seq[pd_index].devHandle;
4298                 } else
4299                         sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"
4300                                 " without JBOD MAP support from %s %d\n", __func__, __LINE__);
4301         } else {
4302                 device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)
4303                                 + sdev->id;
4304                 devhandle = device_id;
4305         }
4306
4307         return devhandle;
4308 }
4309
4310 /*
4311  * megasas_task_abort_fusion : SCSI task abort function for fusion adapters
4312  * @scmd : pointer to scsi command object
4313  *
4314  * Return SUCCESS, if command aborted else FAILED
4315  */
4316
4317 int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
4318 {
4319         struct megasas_instance *instance;
4320         u16 smid, devhandle;
4321         struct fusion_context *fusion;
4322         int ret;
4323         struct MR_PRIV_DEVICE *mr_device_priv_data;
4324         mr_device_priv_data = scmd->device->hostdata;
4325
4326
4327         instance = (struct megasas_instance *)scmd->device->host->hostdata;
4328         fusion = instance->ctrl_context;
4329
4330         scmd_printk(KERN_INFO, scmd, "task abort called for scmd(%p)\n", scmd);
4331         scsi_print_command(scmd);
4332
4333         if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4334                 dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4335                 "SCSI host:%d\n", instance->host->host_no);
4336                 ret = FAILED;
4337                 return ret;
4338         }
4339
4340         if (!mr_device_priv_data) {
4341                 sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4342                         "scmd(%p)\n", scmd);
4343                 scmd->result = DID_NO_CONNECT << 16;
4344                 ret = SUCCESS;
4345                 goto out;
4346         }
4347
4348
4349         if (!mr_device_priv_data->is_tm_capable) {
4350                 ret = FAILED;
4351                 goto out;
4352         }
4353
4354         mutex_lock(&instance->reset_mutex);
4355
4356         smid = megasas_fusion_smid_lookup(scmd);
4357
4358         if (!smid) {
4359                 ret = SUCCESS;
4360                 scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
4361                         " issued is not found in oustanding commands\n");
4362                 mutex_unlock(&instance->reset_mutex);
4363                 goto out;
4364         }
4365
4366         devhandle = megasas_get_tm_devhandle(scmd->device);
4367
4368         if (devhandle == (u16)ULONG_MAX) {
4369                 ret = SUCCESS;
4370                 sdev_printk(KERN_INFO, scmd->device,
4371                         "task abort issued for invalid devhandle\n");
4372                 mutex_unlock(&instance->reset_mutex);
4373                 goto out;
4374         }
4375         sdev_printk(KERN_INFO, scmd->device,
4376                 "attempting task abort! scmd(%p) tm_dev_handle 0x%x\n",
4377                 scmd, devhandle);
4378
4379         mr_device_priv_data->tm_busy = 1;
4380         ret = megasas_issue_tm(instance, devhandle,
4381                         scmd->device->channel, scmd->device->id, smid,
4382                         MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK);
4383         mr_device_priv_data->tm_busy = 0;
4384
4385         mutex_unlock(&instance->reset_mutex);
4386 out:
4387         sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(%p)\n",
4388                         ((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4389
4390         return ret;
4391 }
4392
4393 /*
4394  * megasas_reset_target_fusion : target reset function for fusion adapters
4395  * scmd: SCSI command pointer
4396  *
4397  * Returns SUCCESS if all commands associated with target aborted else FAILED
4398  */
4399
4400 int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
4401 {
4402
4403         struct megasas_instance *instance;
4404         int ret = FAILED;
4405         u16 devhandle;
4406         struct fusion_context *fusion;
4407         struct MR_PRIV_DEVICE *mr_device_priv_data;
4408         mr_device_priv_data = scmd->device->hostdata;
4409
4410         instance = (struct megasas_instance *)scmd->device->host->hostdata;
4411         fusion = instance->ctrl_context;
4412
4413         sdev_printk(KERN_INFO, scmd->device,
4414                     "target reset called for scmd(%p)\n", scmd);
4415
4416         if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4417                 dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4418                 "SCSI host:%d\n", instance->host->host_no);
4419                 ret = FAILED;
4420                 return ret;
4421         }
4422
4423         if (!mr_device_priv_data) {
4424                 sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4425                         "scmd(%p)\n", scmd);
4426                 scmd->result = DID_NO_CONNECT << 16;
4427                 ret = SUCCESS;
4428                 goto out;
4429         }
4430
4431
4432         if (!mr_device_priv_data->is_tm_capable) {
4433                 ret = FAILED;
4434                 goto out;
4435         }
4436
4437         mutex_lock(&instance->reset_mutex);
4438         devhandle = megasas_get_tm_devhandle(scmd->device);
4439
4440         if (devhandle == (u16)ULONG_MAX) {
4441                 ret = SUCCESS;
4442                 sdev_printk(KERN_INFO, scmd->device,
4443                         "target reset issued for invalid devhandle\n");
4444                 mutex_unlock(&instance->reset_mutex);
4445                 goto out;
4446         }
4447
4448         sdev_printk(KERN_INFO, scmd->device,
4449                 "attempting target reset! scmd(%p) tm_dev_handle 0x%x\n",
4450                 scmd, devhandle);
4451         mr_device_priv_data->tm_busy = 1;
4452         ret = megasas_issue_tm(instance, devhandle,
4453                         scmd->device->channel, scmd->device->id, 0,
4454                         MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
4455         mr_device_priv_data->tm_busy = 0;
4456         mutex_unlock(&instance->reset_mutex);
4457 out:
4458         scmd_printk(KERN_NOTICE, scmd, "megasas: target reset %s!!\n",
4459                 (ret == SUCCESS) ? "SUCCESS" : "FAILED");
4460
4461         return ret;
4462 }
4463
4464 /*SRIOV get other instance in cluster if any*/
4465 struct megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)
4466 {
4467         int i;
4468
4469         for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {
4470                 if (megasas_mgmt_info.instance[i] &&
4471                         (megasas_mgmt_info.instance[i] != instance) &&
4472                          megasas_mgmt_info.instance[i]->requestorId &&
4473                          megasas_mgmt_info.instance[i]->peerIsPresent &&
4474                         (memcmp((megasas_mgmt_info.instance[i]->clusterId),
4475                         instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))
4476                         return megasas_mgmt_info.instance[i];
4477         }
4478         return NULL;
4479 }
4480
4481 /* Check for a second path that is currently UP */
4482 int megasas_check_mpio_paths(struct megasas_instance *instance,
4483         struct scsi_cmnd *scmd)
4484 {
4485         struct megasas_instance *peer_instance = NULL;
4486         int retval = (DID_REQUEUE << 16);
4487
4488         if (instance->peerIsPresent) {
4489                 peer_instance = megasas_get_peer_instance(instance);
4490                 if ((peer_instance) &&
4491                         (atomic_read(&peer_instance->adprecovery) ==
4492                         MEGASAS_HBA_OPERATIONAL))
4493                         retval = (DID_NO_CONNECT << 16);
4494         }
4495         return retval;
4496 }
4497
4498 /* Core fusion reset function */
4499 int megasas_reset_fusion(struct Scsi_Host *shost, int reason)
4500 {
4501         int retval = SUCCESS, i, j, convert = 0;
4502         struct megasas_instance *instance;
4503         struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;
4504         struct fusion_context *fusion;
4505         u32 abs_state, status_reg, reset_adapter;
4506         u32 io_timeout_in_crash_mode = 0;
4507         struct scsi_cmnd *scmd_local = NULL;
4508         struct scsi_device *sdev;
4509
4510         instance = (struct megasas_instance *)shost->hostdata;
4511         fusion = instance->ctrl_context;
4512
4513         mutex_lock(&instance->reset_mutex);
4514
4515         if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
4516                 dev_warn(&instance->pdev->dev, "Hardware critical error, "
4517                        "returning FAILED for scsi%d.\n",
4518                         instance->host->host_no);
4519                 mutex_unlock(&instance->reset_mutex);
4520                 return FAILED;
4521         }
4522         status_reg = instance->instancet->read_fw_status_reg(instance->reg_set);
4523         abs_state = status_reg & MFI_STATE_MASK;
4524
4525         /* IO timeout detected, forcibly put FW in FAULT state */
4526         if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&
4527                 instance->crash_dump_app_support && reason) {
4528                 dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "
4529                         "forcibly FAULT Firmware\n");
4530                 atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4531                 status_reg = readl(&instance->reg_set->doorbell);
4532                 writel(status_reg | MFI_STATE_FORCE_OCR,
4533                         &instance->reg_set->doorbell);
4534                 readl(&instance->reg_set->doorbell);
4535                 mutex_unlock(&instance->reset_mutex);
4536                 do {
4537                         ssleep(3);
4538                         io_timeout_in_crash_mode++;
4539                         dev_dbg(&instance->pdev->dev, "waiting for [%d] "
4540                                 "seconds for crash dump collection and OCR "
4541                                 "to be done\n", (io_timeout_in_crash_mode * 3));
4542                 } while ((atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&
4543                         (io_timeout_in_crash_mode < 80));
4544
4545                 if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {
4546                         dev_info(&instance->pdev->dev, "OCR done for IO "
4547                                 "timeout case\n");
4548                         retval = SUCCESS;
4549                 } else {
4550                         dev_info(&instance->pdev->dev, "Controller is not "
4551                                 "operational after 240 seconds wait for IO "
4552                                 "timeout case in FW crash dump mode\n do "
4553                                 "OCR/kill adapter\n");
4554                         retval = megasas_reset_fusion(shost, 0);
4555                 }
4556                 return retval;
4557         }
4558
4559         if (instance->requestorId && !instance->skip_heartbeat_timer_del)
4560                 del_timer_sync(&instance->sriov_heartbeat_timer);
4561         set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4562         atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING);
4563         instance->instancet->disable_intr(instance);
4564         megasas_sync_irqs((unsigned long)instance);
4565
4566         /* First try waiting for commands to complete */
4567         if (megasas_wait_for_outstanding_fusion(instance, reason,
4568                                                 &convert)) {
4569                 atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4570                 dev_warn(&instance->pdev->dev, "resetting fusion "
4571                        "adapter scsi%d.\n", instance->host->host_no);
4572                 if (convert)
4573                         reason = 0;
4574
4575                 if (megasas_dbg_lvl & OCR_LOGS)
4576                         dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");
4577
4578                 /* Now return commands back to the OS */
4579                 for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4580                         cmd_fusion = fusion->cmd_list[i];
4581                         /*check for extra commands issued by driver*/
4582                         if (instance->adapter_type == VENTURA_SERIES) {
4583                                 r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];
4584                                 megasas_return_cmd_fusion(instance, r1_cmd);
4585                         }
4586                         scmd_local = cmd_fusion->scmd;
4587                         if (cmd_fusion->scmd) {
4588                                 if (megasas_dbg_lvl & OCR_LOGS) {
4589                                         sdev_printk(KERN_INFO,
4590                                                 cmd_fusion->scmd->device, "SMID: 0x%x\n",
4591                                                 cmd_fusion->index);
4592                                         scsi_print_command(cmd_fusion->scmd);
4593                                 }
4594
4595                                 scmd_local->result =
4596                                         megasas_check_mpio_paths(instance,
4597                                                         scmd_local);
4598                                 if (instance->ldio_threshold &&
4599                                         megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
4600                                         atomic_dec(&instance->ldio_outstanding);
4601                                 megasas_return_cmd_fusion(instance, cmd_fusion);
4602                                 scsi_dma_unmap(scmd_local);
4603                                 scmd_local->scsi_done(scmd_local);
4604                         }
4605                 }
4606
4607                 atomic_set(&instance->fw_outstanding, 0);
4608
4609                 status_reg = instance->instancet->read_fw_status_reg(
4610                         instance->reg_set);
4611                 abs_state = status_reg & MFI_STATE_MASK;
4612                 reset_adapter = status_reg & MFI_RESET_ADAPTER;
4613                 if (instance->disableOnlineCtrlReset ||
4614                     (abs_state == MFI_STATE_FAULT && !reset_adapter)) {
4615                         /* Reset not supported, kill adapter */
4616                         dev_warn(&instance->pdev->dev, "Reset not supported"
4617                                ", killing adapter scsi%d.\n",
4618                                 instance->host->host_no);
4619                         megaraid_sas_kill_hba(instance);
4620                         instance->skip_heartbeat_timer_del = 1;
4621                         retval = FAILED;
4622                         goto out;
4623                 }
4624
4625                 /* Let SR-IOV VF & PF sync up if there was a HB failure */
4626                 if (instance->requestorId && !reason) {
4627                         msleep(MEGASAS_OCR_SETTLE_TIME_VF);
4628                         goto transition_to_ready;
4629                 }
4630
4631                 /* Now try to reset the chip */
4632                 for (i = 0; i < MEGASAS_FUSION_MAX_RESET_TRIES; i++) {
4633
4634                         if (instance->instancet->adp_reset
4635                                 (instance, instance->reg_set))
4636                                 continue;
4637 transition_to_ready:
4638                         /* Wait for FW to become ready */
4639                         if (megasas_transition_to_ready(instance, 1)) {
4640                                 dev_warn(&instance->pdev->dev,
4641                                         "Failed to transition controller to ready for "
4642                                         "scsi%d.\n", instance->host->host_no);
4643                                 if (instance->requestorId && !reason)
4644                                         goto fail_kill_adapter;
4645                                 else
4646                                         continue;
4647                         }
4648                         megasas_reset_reply_desc(instance);
4649                         megasas_fusion_update_can_queue(instance, OCR_CONTEXT);
4650
4651                         if (megasas_ioc_init_fusion(instance)) {
4652                                 if (instance->requestorId && !reason)
4653                                         goto fail_kill_adapter;
4654                                 else
4655                                         continue;
4656                         }
4657
4658                         if (megasas_get_ctrl_info(instance)) {
4659                                 dev_info(&instance->pdev->dev,
4660                                         "Failed from %s %d\n",
4661                                         __func__, __LINE__);
4662                                 megaraid_sas_kill_hba(instance);
4663                                 retval = FAILED;
4664                                 goto out;
4665                         }
4666
4667                         megasas_refire_mgmt_cmd(instance);
4668
4669                         /* Reset load balance info */
4670                         if (fusion->load_balance_info)
4671                                 memset(fusion->load_balance_info, 0,
4672                                        (sizeof(struct LD_LOAD_BALANCE_INFO) *
4673                                        MAX_LOGICAL_DRIVES_EXT));
4674
4675                         if (!megasas_get_map_info(instance))
4676                                 megasas_sync_map_info(instance);
4677
4678                         megasas_setup_jbod_map(instance);
4679
4680                         shost_for_each_device(sdev, shost)
4681                                 megasas_set_dynamic_target_properties(sdev);
4682
4683                         /* reset stream detection array */
4684                         if (instance->adapter_type == VENTURA_SERIES) {
4685                                 for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {
4686                                         memset(fusion->stream_detect_by_ld[j],
4687                                         0, sizeof(struct LD_STREAM_DETECT));
4688                                  fusion->stream_detect_by_ld[j]->mru_bit_map
4689                                                 = MR_STREAM_BITMAP;
4690                                 }
4691                         }
4692
4693                         clear_bit(MEGASAS_FUSION_IN_RESET,
4694                                   &instance->reset_flags);
4695                         instance->instancet->enable_intr(instance);
4696                         atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
4697
4698                         dev_info(&instance->pdev->dev, "Interrupts are enabled and"
4699                                 " controller is OPERATIONAL for scsi:%d\n",
4700                                 instance->host->host_no);
4701
4702                         /* Restart SR-IOV heartbeat */
4703                         if (instance->requestorId) {
4704                                 if (!megasas_sriov_start_heartbeat(instance, 0))
4705                                         megasas_start_timer(instance);
4706                                 else
4707                                         instance->skip_heartbeat_timer_del = 1;
4708                         }
4709
4710                         if (instance->crash_dump_drv_support &&
4711                                 instance->crash_dump_app_support)
4712                                 megasas_set_crash_dump_params(instance,
4713                                         MR_CRASH_BUF_TURN_ON);
4714                         else
4715                                 megasas_set_crash_dump_params(instance,
4716                                         MR_CRASH_BUF_TURN_OFF);
4717
4718                         retval = SUCCESS;
4719
4720                         /* Adapter reset completed successfully */
4721                         dev_warn(&instance->pdev->dev,
4722                                  "Reset successful for scsi%d.\n",
4723                                  instance->host->host_no);
4724
4725                         goto out;
4726                 }
4727 fail_kill_adapter:
4728                 /* Reset failed, kill the adapter */
4729                 dev_warn(&instance->pdev->dev, "Reset failed, killing "
4730                        "adapter scsi%d.\n", instance->host->host_no);
4731                 megaraid_sas_kill_hba(instance);
4732                 instance->skip_heartbeat_timer_del = 1;
4733                 retval = FAILED;
4734         } else {
4735                 /* For VF: Restart HB timer if we didn't OCR */
4736                 if (instance->requestorId) {
4737                         megasas_start_timer(instance);
4738                 }
4739                 clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4740                 instance->instancet->enable_intr(instance);
4741                 atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
4742         }
4743 out:
4744         clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4745         mutex_unlock(&instance->reset_mutex);
4746         return retval;
4747 }
4748
4749 /* Fusion Crash dump collection work queue */
4750 void  megasas_fusion_crash_dump_wq(struct work_struct *work)
4751 {
4752         struct megasas_instance *instance =
4753                 container_of(work, struct megasas_instance, crash_init);
4754         u32 status_reg;
4755         u8 partial_copy = 0;
4756
4757
4758         status_reg = instance->instancet->read_fw_status_reg(instance->reg_set);
4759
4760         /*
4761          * Allocate host crash buffers to copy data from 1 MB DMA crash buffer
4762          * to host crash buffers
4763          */
4764         if (instance->drv_buf_index == 0) {
4765                 /* Buffer is already allocated for old Crash dump.
4766                  * Do OCR and do not wait for crash dump collection
4767                  */
4768                 if (instance->drv_buf_alloc) {
4769                         dev_info(&instance->pdev->dev, "earlier crash dump is "
4770                                 "not yet copied by application, ignoring this "
4771                                 "crash dump and initiating OCR\n");
4772                         status_reg |= MFI_STATE_CRASH_DUMP_DONE;
4773                         writel(status_reg,
4774                                 &instance->reg_set->outbound_scratch_pad);
4775                         readl(&instance->reg_set->outbound_scratch_pad);
4776                         return;
4777                 }
4778                 megasas_alloc_host_crash_buffer(instance);
4779                 dev_info(&instance->pdev->dev, "Number of host crash buffers "
4780                         "allocated: %d\n", instance->drv_buf_alloc);
4781         }
4782
4783         /*
4784          * Driver has allocated max buffers, which can be allocated
4785          * and FW has more crash dump data, then driver will
4786          * ignore the data.
4787          */
4788         if (instance->drv_buf_index >= (instance->drv_buf_alloc)) {
4789                 dev_info(&instance->pdev->dev, "Driver is done copying "
4790                         "the buffer: %d\n", instance->drv_buf_alloc);
4791                 status_reg |= MFI_STATE_CRASH_DUMP_DONE;
4792                 partial_copy = 1;
4793         } else {
4794                 memcpy(instance->crash_buf[instance->drv_buf_index],
4795                         instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);
4796                 instance->drv_buf_index++;
4797                 status_reg &= ~MFI_STATE_DMADONE;
4798         }
4799
4800         if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {
4801                 dev_info(&instance->pdev->dev, "Crash Dump is available,number "
4802                         "of copied buffers: %d\n", instance->drv_buf_index);
4803                 instance->fw_crash_buffer_size =  instance->drv_buf_index;
4804                 instance->fw_crash_state = AVAILABLE;
4805                 instance->drv_buf_index = 0;
4806                 writel(status_reg, &instance->reg_set->outbound_scratch_pad);
4807                 readl(&instance->reg_set->outbound_scratch_pad);
4808                 if (!partial_copy)
4809                         megasas_reset_fusion(instance->host, 0);
4810         } else {
4811                 writel(status_reg, &instance->reg_set->outbound_scratch_pad);
4812                 readl(&instance->reg_set->outbound_scratch_pad);
4813         }
4814 }
4815
4816
4817 /* Fusion OCR work queue */
4818 void megasas_fusion_ocr_wq(struct work_struct *work)
4819 {
4820         struct megasas_instance *instance =
4821                 container_of(work, struct megasas_instance, work_init);
4822
4823         megasas_reset_fusion(instance->host, 0);
4824 }
4825
4826 /* Allocate fusion context */
4827 int
4828 megasas_alloc_fusion_context(struct megasas_instance *instance)
4829 {
4830         struct fusion_context *fusion;
4831
4832         instance->ctrl_context = kzalloc(sizeof(struct fusion_context),
4833                                          GFP_KERNEL);
4834         if (!instance->ctrl_context) {
4835                 dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4836                         __func__, __LINE__);
4837                 return -ENOMEM;
4838         }
4839
4840         fusion = instance->ctrl_context;
4841
4842         fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
4843                                               sizeof(LD_SPAN_INFO));
4844         fusion->log_to_span =
4845                 (PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
4846                                                 fusion->log_to_span_pages);
4847         if (!fusion->log_to_span) {
4848                 fusion->log_to_span = vzalloc(MAX_LOGICAL_DRIVES_EXT *
4849                                               sizeof(LD_SPAN_INFO));
4850                 if (!fusion->log_to_span) {
4851                         dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4852                                 __func__, __LINE__);
4853                         return -ENOMEM;
4854                 }
4855         }
4856
4857         fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
4858                 sizeof(struct LD_LOAD_BALANCE_INFO));
4859         fusion->load_balance_info =
4860                 (struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
4861                 fusion->load_balance_info_pages);
4862         if (!fusion->load_balance_info) {
4863                 fusion->load_balance_info = vzalloc(MAX_LOGICAL_DRIVES_EXT *
4864                         sizeof(struct LD_LOAD_BALANCE_INFO));
4865                 if (!fusion->load_balance_info)
4866                         dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "
4867                                 "continuing without Load Balance support\n");
4868         }
4869
4870         return 0;
4871 }
4872
4873 void
4874 megasas_free_fusion_context(struct megasas_instance *instance)
4875 {
4876         struct fusion_context *fusion = instance->ctrl_context;
4877
4878         if (fusion) {
4879                 if (fusion->load_balance_info) {
4880                         if (is_vmalloc_addr(fusion->load_balance_info))
4881                                 vfree(fusion->load_balance_info);
4882                         else
4883                                 free_pages((ulong)fusion->load_balance_info,
4884                                         fusion->load_balance_info_pages);
4885                 }
4886
4887                 if (fusion->log_to_span) {
4888                         if (is_vmalloc_addr(fusion->log_to_span))
4889                                 vfree(fusion->log_to_span);
4890                         else
4891                                 free_pages((ulong)fusion->log_to_span,
4892                                            fusion->log_to_span_pages);
4893                 }
4894
4895                 kfree(fusion);
4896         }
4897 }
4898
4899 struct megasas_instance_template megasas_instance_template_fusion = {
4900         .enable_intr = megasas_enable_intr_fusion,
4901         .disable_intr = megasas_disable_intr_fusion,
4902         .clear_intr = megasas_clear_intr_fusion,
4903         .read_fw_status_reg = megasas_read_fw_status_reg_fusion,
4904         .adp_reset = megasas_adp_reset_fusion,
4905         .check_reset = megasas_check_reset_fusion,
4906         .service_isr = megasas_isr_fusion,
4907         .tasklet = megasas_complete_cmd_dpc_fusion,
4908         .init_adapter = megasas_init_adapter_fusion,
4909         .build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,
4910         .issue_dcmd = megasas_issue_dcmd_fusion,
4911 };