drivers/scsi/isci/request.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  * redistributing this file, you may do so under either license.
   4  *
   5  * GPL LICENSE SUMMARY
   6  *
   7  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  21  * The full GNU General Public License is included in this distribution
  22  * in the file called LICENSE.GPL.
  23  *
  24  * BSD LICENSE
  25  *
  26  * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved.
  27  * All rights reserved.
  28  *
  29  * Redistribution and use in source and binary forms, with or without
  30  * modification, are permitted provided that the following conditions
  31  * are met:
  32  *
  33  *   * Redistributions of source code must retain the above copyright
  34  *     notice, this list of conditions and the following disclaimer.
  35  *   * Redistributions in binary form must reproduce the above copyright
  36  *     notice, this list of conditions and the following disclaimer in
  37  *     the documentation and/or other materials provided with the
  38  *     distribution.
  39  *   * Neither the name of Intel Corporation nor the names of its
  40  *     contributors may be used to endorse or promote products derived
  41  *     from this software without specific prior written permission.
  42  *
  43  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  44  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  45  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  46  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  47  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  48  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  49  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  50  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  51  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  52  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  53  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  54  */
  55
  56 #include <scsi/scsi_cmnd.h>
  57 #include "isci.h"
  58 #include "task.h"
  59 #include "request.h"
  60 #include "scu_completion_codes.h"
  61 #include "scu_event_codes.h"
  62 #include "sas.h"
  63
  64 #undef C
  65 #define C(a) (#a)
  66 const char *req_state_name(enum sci_base_request_states state)
  67 {
  68         static const char * const strings[] = REQUEST_STATES;
  69
  70         return strings[state];
  71 }
  72 #undef C
  73
  74 static struct scu_sgl_element_pair *to_sgl_element_pair(struct isci_request *ireq,
  75                                                         int idx)
  76 {
  77         if (idx == 0)
  78                 return &ireq->tc->sgl_pair_ab;
  79         else if (idx == 1)
  80                 return &ireq->tc->sgl_pair_cd;
  81         else if (idx < 0)
  82                 return NULL;
  83         else
  84                 return &ireq->sg_table[idx - 2];
  85 }
  86
  87 static dma_addr_t to_sgl_element_pair_dma(struct isci_host *ihost,
  88                                           struct isci_request *ireq, u32 idx)
  89 {
  90         u32 offset;
  91
  92         if (idx == 0) {
  93                 offset = (void *) &ireq->tc->sgl_pair_ab -
  94                          (void *) &ihost->task_context_table[0];
  95                 return ihost->tc_dma + offset;
  96         } else if (idx == 1) {
  97                 offset = (void *) &ireq->tc->sgl_pair_cd -
  98                          (void *) &ihost->task_context_table[0];
  99                 return ihost->tc_dma + offset;
 100         }
 101
 102         return sci_io_request_get_dma_addr(ireq, &ireq->sg_table[idx - 2]);
 103 }
 104
 105 static void init_sgl_element(struct scu_sgl_element *e, struct scatterlist *sg)
 106 {
 107         e->length = sg_dma_len(sg);
 108         e->address_upper = upper_32_bits(sg_dma_address(sg));
 109         e->address_lower = lower_32_bits(sg_dma_address(sg));
 110         e->address_modifier = 0;
 111 }
 112
 113 static void sci_request_build_sgl(struct isci_request *ireq)
 114 {
 115         struct isci_host *ihost = ireq->isci_host;
 116         struct sas_task *task = isci_request_access_task(ireq);
 117         struct scatterlist *sg = NULL;
 118         dma_addr_t dma_addr;
 119         u32 sg_idx = 0;
 120         struct scu_sgl_element_pair *scu_sg   = NULL;
 121         struct scu_sgl_element_pair *prev_sg  = NULL;
 122
 123         if (task->num_scatter > 0) {
 124                 sg = task->scatter;
 125
 126                 while (sg) {
 127                         scu_sg = to_sgl_element_pair(ireq, sg_idx);
 128                         init_sgl_element(&scu_sg->A, sg);
 129                         sg = sg_next(sg);
 130                         if (sg) {
 131                                 init_sgl_element(&scu_sg->B, sg);
 132                                 sg = sg_next(sg);
 133                         } else
 134                                 memset(&scu_sg->B, 0, sizeof(scu_sg->B));
 135
 136                         if (prev_sg) {
 137                                 dma_addr = to_sgl_element_pair_dma(ihost,
 138                                                                    ireq,
 139                                                                    sg_idx);
 140
 141                                 prev_sg->next_pair_upper =
 142                                         upper_32_bits(dma_addr);
 143                                 prev_sg->next_pair_lower =
 144                                         lower_32_bits(dma_addr);
 145                         }
 146
 147                         prev_sg = scu_sg;
 148                         sg_idx++;
 149                 }
 150         } else {        /* handle when no sg */
 151                 scu_sg = to_sgl_element_pair(ireq, sg_idx);
 152
 153                 dma_addr = dma_map_single(&ihost->pdev->dev,
 154                                           task->scatter,
 155                                           task->total_xfer_len,
 156                                           task->data_dir);
 157
 158                 ireq->zero_scatter_daddr = dma_addr;
 159
 160                 scu_sg->A.length = task->total_xfer_len;
 161                 scu_sg->A.address_upper = upper_32_bits(dma_addr);
 162                 scu_sg->A.address_lower = lower_32_bits(dma_addr);
 163         }
 164
 165         if (scu_sg) {
 166                 scu_sg->next_pair_upper = 0;
 167                 scu_sg->next_pair_lower = 0;
 168         }
 169 }
 170
 171 static void sci_io_request_build_ssp_command_iu(struct isci_request *ireq)
 172 {
 173         struct ssp_cmd_iu *cmd_iu;
 174         struct sas_task *task = isci_request_access_task(ireq);
 175
 176         cmd_iu = &ireq->ssp.cmd;
 177
 178         memcpy(cmd_iu->LUN, task->ssp_task.LUN, 8);
 179         cmd_iu->add_cdb_len = 0;
 180         cmd_iu->_r_a = 0;
 181         cmd_iu->_r_b = 0;
 182         cmd_iu->en_fburst = 0; /* unsupported */
 183         cmd_iu->task_prio = task->ssp_task.task_prio;
 184         cmd_iu->task_attr = task->ssp_task.task_attr;
 185         cmd_iu->_r_c = 0;
 186
 187         sci_swab32_cpy(&cmd_iu->cdb, task->ssp_task.cmd->cmnd,
 188                        (task->ssp_task.cmd->cmd_len+3) / sizeof(u32));
 189 }
 190
 191 static void sci_task_request_build_ssp_task_iu(struct isci_request *ireq)
 192 {
 193         struct ssp_task_iu *task_iu;
 194         struct sas_task *task = isci_request_access_task(ireq);
 195         struct isci_tmf *isci_tmf = isci_request_access_tmf(ireq);
 196
 197         task_iu = &ireq->ssp.tmf;
 198
 199         memset(task_iu, 0, sizeof(struct ssp_task_iu));
 200
 201         memcpy(task_iu->LUN, task->ssp_task.LUN, 8);
 202
 203         task_iu->task_func = isci_tmf->tmf_code;
 204         task_iu->task_tag =
 205                 (test_bit(IREQ_TMF, &ireq->flags)) ?
 206                 isci_tmf->io_tag :
 207                 SCI_CONTROLLER_INVALID_IO_TAG;
 208 }
 209
 210 /*
 211  * This method is will fill in the SCU Task Context for any type of SSP request.
 212  */
 213 static void scu_ssp_request_construct_task_context(
 214         struct isci_request *ireq,
 215         struct scu_task_context *task_context)
 216 {
 217         dma_addr_t dma_addr;
 218         struct isci_remote_device *idev;
 219         struct isci_port *iport;
 220
 221         idev = ireq->target_device;
 222         iport = idev->owning_port;
 223
 224         /* Fill in the TC with its required data */
 225         task_context->abort = 0;
 226         task_context->priority = 0;
 227         task_context->initiator_request = 1;
 228         task_context->connection_rate = idev->connection_rate;
 229         task_context->protocol_engine_index = ISCI_PEG;
 230         task_context->logical_port_index = iport->physical_port_index;
 231         task_context->protocol_type = SCU_TASK_CONTEXT_PROTOCOL_SSP;
 232         task_context->valid = SCU_TASK_CONTEXT_VALID;
 233         task_context->context_type = SCU_TASK_CONTEXT_TYPE;
 234
 235         task_context->remote_node_index = idev->rnc.remote_node_index;
 236         task_context->command_code = 0;
 237
 238         task_context->link_layer_control = 0;
 239         task_context->do_not_dma_ssp_good_response = 1;
 240         task_context->strict_ordering = 0;
 241         task_context->control_frame = 0;
 242         task_context->timeout_enable = 0;
 243         task_context->block_guard_enable = 0;
 244
 245         task_context->address_modifier = 0;
 246
 247         /* task_context->type.ssp.tag = ireq->io_tag; */
 248         task_context->task_phase = 0x01;
 249
 250         ireq->post_context = (SCU_CONTEXT_COMMAND_REQUEST_TYPE_POST_TC |
 251                               (ISCI_PEG << SCU_CONTEXT_COMMAND_PROTOCOL_ENGINE_GROUP_SHIFT) |
 252                               (iport->physical_port_index <<
 253                                SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT) |
 254                               ISCI_TAG_TCI(ireq->io_tag));
 255
 256         /*
 257          * Copy the physical address for the command buffer to the
 258          * SCU Task Context
 259          */
 260         dma_addr = sci_io_request_get_dma_addr(ireq, &ireq->ssp.cmd);
 261
 262         task_context->command_iu_upper = upper_32_bits(dma_addr);
 263         task_context->command_iu_lower = lower_32_bits(dma_addr);
 264
 265         /*
 266          * Copy the physical address for the response buffer to the
 267          * SCU Task Context
 268          */
 269         dma_addr = sci_io_request_get_dma_addr(ireq, &ireq->ssp.rsp);
 270
 271         task_context->response_iu_upper = upper_32_bits(dma_addr);
 272         task_context->response_iu_lower = lower_32_bits(dma_addr);
 273 }
 274
 275 static u8 scu_bg_blk_size(struct scsi_device *sdp)
 276 {
 277         switch (sdp->sector_size) {
 278         case 512:
 279                 return 0;
 280         case 1024:
 281                 return 1;
 282         case 4096:
 283                 return 3;
 284         default:
 285                 return 0xff;
 286         }
 287 }
 288
 289 static u32 scu_dif_bytes(u32 len, u32 sector_size)
 290 {
 291         return (len >> ilog2(sector_size)) * 8;
 292 }
 293
 294 static void scu_ssp_ireq_dif_insert(struct isci_request *ireq, u8 type, u8 op)
 295 {
 296         struct scu_task_context *tc = ireq->tc;
 297         struct scsi_cmnd *scmd = ireq->ttype_ptr.io_task_ptr->uldd_task;
 298         u8 blk_sz = scu_bg_blk_size(scmd->device);
 299
 300         tc->block_guard_enable = 1;
 301         tc->blk_prot_en = 1;
 302         tc->blk_sz = blk_sz;
 303         /* DIF write insert */
 304         tc->blk_prot_func = 0x2;
 305
 306         tc->transfer_length_bytes += scu_dif_bytes(tc->transfer_length_bytes,
 307                                                    scmd->device->sector_size);
 308
 309         /* always init to 0, used by hw */
 310         tc->interm_crc_val = 0;
 311
 312         tc->init_crc_seed = 0;
 313         tc->app_tag_verify = 0;
 314         tc->app_tag_gen = 0;
 315         tc->ref_tag_seed_verify = 0;
 316
 317         /* always init to same as bg_blk_sz */
 318         tc->UD_bytes_immed_val = scmd->device->sector_size;
 319
 320         tc->reserved_DC_0 = 0;
 321
 322         /* always init to 8 */
 323         tc->DIF_bytes_immed_val = 8;
 324
 325         tc->reserved_DC_1 = 0;
 326         tc->bgc_blk_sz = scmd->device->sector_size;
 327         tc->reserved_E0_0 = 0;
 328         tc->app_tag_gen_mask = 0;
 329
 330         /** setup block guard control **/
 331         tc->bgctl = 0;
 332
 333         /* DIF write insert */
 334         tc->bgctl_f.op = 0x2;
 335
 336         tc->app_tag_verify_mask = 0;
 337
 338         /* must init to 0 for hw */
 339         tc->blk_guard_err = 0;
 340
 341         tc->reserved_E8_0 = 0;
 342
 343         if ((type & SCSI_PROT_DIF_TYPE1) || (type & SCSI_PROT_DIF_TYPE2))
 344                 tc->ref_tag_seed_gen = scsi_prot_ref_tag(scmd);
 345         else if (type & SCSI_PROT_DIF_TYPE3)
 346                 tc->ref_tag_seed_gen = 0;
 347 }
 348
 349 static void scu_ssp_ireq_dif_strip(struct isci_request *ireq, u8 type, u8 op)
 350 {
 351         struct scu_task_context *tc = ireq->tc;
 352         struct scsi_cmnd *scmd = ireq->ttype_ptr.io_task_ptr->uldd_task;
 353         u8 blk_sz = scu_bg_blk_size(scmd->device);
 354
 355         tc->block_guard_enable = 1;
 356         tc->blk_prot_en = 1;
 357         tc->blk_sz = blk_sz;
 358         /* DIF read strip */
 359         tc->blk_prot_func = 0x1;
 360
 361         tc->transfer_length_bytes += scu_dif_bytes(tc->transfer_length_bytes,
 362                                                    scmd->device->sector_size);
 363
 364         /* always init to 0, used by hw */
 365         tc->interm_crc_val = 0;
 366
 367         tc->init_crc_seed = 0;
 368         tc->app_tag_verify = 0;
 369         tc->app_tag_gen = 0;
 370
 371         if ((type & SCSI_PROT_DIF_TYPE1) || (type & SCSI_PROT_DIF_TYPE2))
 372                 tc->ref_tag_seed_verify = scsi_prot_ref_tag(scmd);
 373         else if (type & SCSI_PROT_DIF_TYPE3)
 374                 tc->ref_tag_seed_verify = 0;
 375
 376         /* always init to same as bg_blk_sz */
 377         tc->UD_bytes_immed_val = scmd->device->sector_size;
 378
 379         tc->reserved_DC_0 = 0;
 380
 381         /* always init to 8 */
 382         tc->DIF_bytes_immed_val = 8;
 383
 384         tc->reserved_DC_1 = 0;
 385         tc->bgc_blk_sz = scmd->device->sector_size;
 386         tc->reserved_E0_0 = 0;
 387         tc->app_tag_gen_mask = 0;
 388
 389         /** setup block guard control **/
 390         tc->bgctl = 0;
 391
 392         /* DIF read strip */
 393         tc->bgctl_f.crc_verify = 1;
 394         tc->bgctl_f.op = 0x1;
 395         if ((type & SCSI_PROT_DIF_TYPE1) || (type & SCSI_PROT_DIF_TYPE2)) {
 396                 tc->bgctl_f.ref_tag_chk = 1;
 397                 tc->bgctl_f.app_f_detect = 1;
 398         } else if (type & SCSI_PROT_DIF_TYPE3)
 399                 tc->bgctl_f.app_ref_f_detect = 1;
 400
 401         tc->app_tag_verify_mask = 0;
 402
 403         /* must init to 0 for hw */
 404         tc->blk_guard_err = 0;
 405
 406         tc->reserved_E8_0 = 0;
 407         tc->ref_tag_seed_gen = 0;
 408 }
 409
 410 /*
 411  * This method is will fill in the SCU Task Context for a SSP IO request.
 412  */
 413 static void scu_ssp_io_request_construct_task_context(struct isci_request *ireq,
 414                                                       enum dma_data_direction dir,
 415                                                       u32 len)
 416 {
 417         struct scu_task_context *task_context = ireq->tc;
 418         struct sas_task *sas_task = ireq->ttype_ptr.io_task_ptr;
 419         struct scsi_cmnd *scmd = sas_task->uldd_task;
 420         u8 prot_type = scsi_get_prot_type(scmd);
 421         u8 prot_op = scsi_get_prot_op(scmd);
 422
 423         scu_ssp_request_construct_task_context(ireq, task_context);
 424
 425         task_context->ssp_command_iu_length =
 426                 sizeof(struct ssp_cmd_iu) / sizeof(u32);
 427         task_context->type.ssp.frame_type = SSP_COMMAND;
 428
 429         switch (dir) {
 430         case DMA_FROM_DEVICE:
 431         case DMA_NONE:
 432         default:
 433                 task_context->task_type = SCU_TASK_TYPE_IOREAD;
 434                 break;
 435         case DMA_TO_DEVICE:
 436                 task_context->task_type = SCU_TASK_TYPE_IOWRITE;
 437                 break;
 438         }
 439
 440         task_context->transfer_length_bytes = len;
 441
 442         if (task_context->transfer_length_bytes > 0)
 443                 sci_request_build_sgl(ireq);
 444
 445         if (prot_type != SCSI_PROT_DIF_TYPE0) {
 446                 if (prot_op == SCSI_PROT_READ_STRIP)
 447                         scu_ssp_ireq_dif_strip(ireq, prot_type, prot_op);
 448                 else if (prot_op == SCSI_PROT_WRITE_INSERT)
 449                         scu_ssp_ireq_dif_insert(ireq, prot_type, prot_op);
 450         }
 451 }
 452
 453 /**
 454  * scu_ssp_task_request_construct_task_context() - This method will fill in
 455  *    the SCU Task Context for a SSP Task request.  The following important
 456  *    settings are utilized: -# priority == SCU_TASK_PRIORITY_HIGH.  This
 457  *    ensures that the task request is issued ahead of other task destined
 458  *    for the same Remote Node. -# task_type == SCU_TASK_TYPE_IOREAD.  This
 459  *    simply indicates that a normal request type (i.e. non-raw frame) is
 460  *    being utilized to perform task management. -#control_frame == 1.  This
 461  *    ensures that the proper endianness is set so that the bytes are
 462  *    transmitted in the right order for a task frame.
 463  * @ireq: This parameter specifies the task request object being constructed.
 464  */
 465 static void scu_ssp_task_request_construct_task_context(struct isci_request *ireq)
 466 {
 467         struct scu_task_context *task_context = ireq->tc;
 468
 469         scu_ssp_request_construct_task_context(ireq, task_context);
 470
 471         task_context->control_frame                = 1;
 472         task_context->priority                     = SCU_TASK_PRIORITY_HIGH;
 473         task_context->task_type                    = SCU_TASK_TYPE_RAW_FRAME;
 474         task_context->transfer_length_bytes        = 0;
 475         task_context->type.ssp.frame_type          = SSP_TASK;
 476         task_context->ssp_command_iu_length =
 477                 sizeof(struct ssp_task_iu) / sizeof(u32);
 478 }
 479
 480 /**
 481  * scu_sata_request_construct_task_context()
 482  * This method is will fill in the SCU Task Context for any type of SATA
 483  *    request.  This is called from the various SATA constructors.
 484  * @ireq: The general IO request object which is to be used in
 485  *    constructing the SCU task context.
 486  * @task_context: The buffer pointer for the SCU task context which is being
 487  *    constructed.
 488  *
 489  * The general io request construction is complete. The buffer assignment for
 490  * the command buffer is complete. none Revisit task context construction to
 491  * determine what is common for SSP/SMP/STP task context structures.
 492  */
 493 static void scu_sata_request_construct_task_context(
 494         struct isci_request *ireq,
 495         struct scu_task_context *task_context)
 496 {
 497         dma_addr_t dma_addr;
 498         struct isci_remote_device *idev;
 499         struct isci_port *iport;
 500
 501         idev = ireq->target_device;
 502         iport = idev->owning_port;
 503
 504         /* Fill in the TC with its required data */
 505         task_context->abort = 0;
 506         task_context->priority = SCU_TASK_PRIORITY_NORMAL;
 507         task_context->initiator_request = 1;
 508         task_context->connection_rate = idev->connection_rate;
 509         task_context->protocol_engine_index = ISCI_PEG;
 510         task_context->logical_port_index = iport->physical_port_index;
 511         task_context->protocol_type = SCU_TASK_CONTEXT_PROTOCOL_STP;
 512         task_context->valid = SCU_TASK_CONTEXT_VALID;
 513         task_context->context_type = SCU_TASK_CONTEXT_TYPE;
 514
 515         task_context->remote_node_index = idev->rnc.remote_node_index;
 516         task_context->command_code = 0;
 517
 518         task_context->link_layer_control = 0;
 519         task_context->do_not_dma_ssp_good_response = 1;
 520         task_context->strict_ordering = 0;
 521         task_context->control_frame = 0;
 522         task_context->timeout_enable = 0;
 523         task_context->block_guard_enable = 0;
 524
 525         task_context->address_modifier = 0;
 526         task_context->task_phase = 0x01;
 527
 528         task_context->ssp_command_iu_length =
 529                 (sizeof(struct host_to_dev_fis) - sizeof(u32)) / sizeof(u32);
 530
 531         /* Set the first word of the H2D REG FIS */
 532         task_context->type.words[0] = *(u32 *)&ireq->stp.cmd;
 533
 534         ireq->post_context = (SCU_CONTEXT_COMMAND_REQUEST_TYPE_POST_TC |
 535                               (ISCI_PEG << SCU_CONTEXT_COMMAND_PROTOCOL_ENGINE_GROUP_SHIFT) |
 536                               (iport->physical_port_index <<
 537                                SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT) |
 538                               ISCI_TAG_TCI(ireq->io_tag));
 539         /*
 540          * Copy the physical address for the command buffer to the SCU Task
 541          * Context. We must offset the command buffer by 4 bytes because the
 542          * first 4 bytes are transfered in the body of the TC.
 543          */
 544         dma_addr = sci_io_request_get_dma_addr(ireq,
 545                                                 ((char *) &ireq->stp.cmd) +
 546                                                 sizeof(u32));
 547
 548         task_context->command_iu_upper = upper_32_bits(dma_addr);
 549         task_context->command_iu_lower = lower_32_bits(dma_addr);
 550
 551         /* SATA Requests do not have a response buffer */
 552         task_context->response_iu_upper = 0;
 553         task_context->response_iu_lower = 0;
 554 }
 555
 556 static void scu_stp_raw_request_construct_task_context(struct isci_request *ireq)
 557 {
 558         struct scu_task_context *task_context = ireq->tc;
 559
 560         scu_sata_request_construct_task_context(ireq, task_context);
 561
 562         task_context->control_frame         = 0;
 563         task_context->priority              = SCU_TASK_PRIORITY_NORMAL;
 564         task_context->task_type             = SCU_TASK_TYPE_SATA_RAW_FRAME;
 565         task_context->type.stp.fis_type     = FIS_REGH2D;
 566         task_context->transfer_length_bytes = sizeof(struct host_to_dev_fis) - sizeof(u32);
 567 }
 568
 569 static enum sci_status sci_stp_pio_request_construct(struct isci_request *ireq,
 570                                                           bool copy_rx_frame)
 571 {
 572         struct isci_stp_request *stp_req = &ireq->stp.req;
 573
 574         scu_stp_raw_request_construct_task_context(ireq);
 575
 576         stp_req->status = 0;
 577         stp_req->sgl.offset = 0;
 578         stp_req->sgl.set = SCU_SGL_ELEMENT_PAIR_A;
 579
 580         if (copy_rx_frame) {
 581                 sci_request_build_sgl(ireq);
 582                 stp_req->sgl.index = 0;
 583         } else {
 584                 /* The user does not want the data copied to the SGL buffer location */
 585                 stp_req->sgl.index = -1;
 586         }
 587
 588         return SCI_SUCCESS;
 589 }
 590
 591 /*
 592  * sci_stp_optimized_request_construct()
 593  * @ireq: This parameter specifies the request to be constructed as an
 594  *    optimized request.
 595  * @optimized_task_type: This parameter specifies whether the request is to be
 596  *    an UDMA request or a NCQ request. - A value of 0 indicates UDMA. - A
 597  *    value of 1 indicates NCQ.
 598  *
 599  * This method will perform request construction common to all types of STP
 600  * requests that are optimized by the silicon (i.e. UDMA, NCQ). This method
 601  * returns an indication as to whether the construction was successful.
 602  */
 603 static void sci_stp_optimized_request_construct(struct isci_request *ireq,
 604                                                      u8 optimized_task_type,
 605                                                      u32 len,
 606                                                      enum dma_data_direction dir)
 607 {
 608         struct scu_task_context *task_context = ireq->tc;
 609
 610         /* Build the STP task context structure */
 611         scu_sata_request_construct_task_context(ireq, task_context);
 612
 613         /* Copy over the SGL elements */
 614         sci_request_build_sgl(ireq);
 615
 616         /* Copy over the number of bytes to be transfered */
 617         task_context->transfer_length_bytes = len;
 618
 619         if (dir == DMA_TO_DEVICE) {
 620                 /*
 621                  * The difference between the DMA IN and DMA OUT request task type
 622                  * values are consistent with the difference between FPDMA READ
 623                  * and FPDMA WRITE values.  Add the supplied task type parameter
 624                  * to this difference to set the task type properly for this
 625                  * DATA OUT (WRITE) case. */
 626                 task_context->task_type = optimized_task_type + (SCU_TASK_TYPE_DMA_OUT
 627                                                                  - SCU_TASK_TYPE_DMA_IN);
 628         } else {
 629                 /*
 630                  * For the DATA IN (READ) case, simply save the supplied
 631                  * optimized task type. */
 632                 task_context->task_type = optimized_task_type;
 633         }
 634 }
 635
 636 static void sci_atapi_construct(struct isci_request *ireq)
 637 {
 638         struct host_to_dev_fis *h2d_fis = &ireq->stp.cmd;
 639         struct sas_task *task;
 640
 641         /* To simplify the implementation we take advantage of the
 642          * silicon's partial acceleration of atapi protocol (dma data
 643          * transfers), so we promote all commands to dma protocol.  This
 644          * breaks compatibility with ATA_HORKAGE_ATAPI_MOD16_DMA drives.
 645          */
 646         h2d_fis->features |= ATAPI_PKT_DMA;
 647
 648         scu_stp_raw_request_construct_task_context(ireq);
 649
 650         task = isci_request_access_task(ireq);
 651         if (task->data_dir == DMA_NONE)
 652                 task->total_xfer_len = 0;
 653
 654         /* clear the response so we can detect arrivial of an
 655          * unsolicited h2d fis
 656          */
 657         ireq->stp.rsp.fis_type = 0;
 658 }
 659
 660 static enum sci_status
 661 sci_io_request_construct_sata(struct isci_request *ireq,
 662                                u32 len,
 663                                enum dma_data_direction dir,
 664                                bool copy)
 665 {
 666         enum sci_status status = SCI_SUCCESS;
 667         struct sas_task *task = isci_request_access_task(ireq);
 668         struct domain_device *dev = ireq->target_device->domain_dev;
 669
 670         /* check for management protocols */
 671         if (test_bit(IREQ_TMF, &ireq->flags)) {
 672                 struct isci_tmf *tmf = isci_request_access_tmf(ireq);
 673
 674                 dev_err(&ireq->owning_controller->pdev->dev,
 675                         "%s: Request 0x%p received un-handled SAT "
 676                         "management protocol 0x%x.\n",
 677                         __func__, ireq, tmf->tmf_code);
 678
 679                 return SCI_FAILURE;
 680         }
 681
 682         if (!sas_protocol_ata(task->task_proto)) {
 683                 dev_err(&ireq->owning_controller->pdev->dev,
 684                         "%s: Non-ATA protocol in SATA path: 0x%x\n",
 685                         __func__,
 686                         task->task_proto);
 687                 return SCI_FAILURE;
 688
 689         }
 690
 691         /* ATAPI */
 692         if (dev->sata_dev.class == ATA_DEV_ATAPI &&
 693             task->ata_task.fis.command == ATA_CMD_PACKET) {
 694                 sci_atapi_construct(ireq);
 695                 return SCI_SUCCESS;
 696         }
 697
 698         /* non data */
 699         if (task->data_dir == DMA_NONE) {
 700                 scu_stp_raw_request_construct_task_context(ireq);
 701                 return SCI_SUCCESS;
 702         }
 703
 704         /* NCQ */
 705         if (task->ata_task.use_ncq) {
 706                 sci_stp_optimized_request_construct(ireq,
 707                                                          SCU_TASK_TYPE_FPDMAQ_READ,
 708                                                          len, dir);
 709                 return SCI_SUCCESS;
 710         }
 711
 712         /* DMA */
 713         if (task->ata_task.dma_xfer) {
 714                 sci_stp_optimized_request_construct(ireq,
 715                                                          SCU_TASK_TYPE_DMA_IN,
 716                                                          len, dir);
 717                 return SCI_SUCCESS;
 718         } else /* PIO */
 719                 return sci_stp_pio_request_construct(ireq, copy);
 720
 721         return status;
 722 }
 723
 724 static enum sci_status sci_io_request_construct_basic_ssp(struct isci_request *ireq)
 725 {
 726         struct sas_task *task = isci_request_access_task(ireq);
 727
 728         ireq->protocol = SAS_PROTOCOL_SSP;
 729
 730         scu_ssp_io_request_construct_task_context(ireq,
 731                                                   task->data_dir,
 732                                                   task->total_xfer_len);
 733
 734         sci_io_request_build_ssp_command_iu(ireq);
 735
 736         sci_change_state(&ireq->sm, SCI_REQ_CONSTRUCTED);
 737
 738         return SCI_SUCCESS;
 739 }
 740
 741 enum sci_status sci_task_request_construct_ssp(
 742         struct isci_request *ireq)
 743 {
 744         /* Construct the SSP Task SCU Task Context */
 745         scu_ssp_task_request_construct_task_context(ireq);
 746
 747         /* Fill in the SSP Task IU */
 748         sci_task_request_build_ssp_task_iu(ireq);
 749
 750         sci_change_state(&ireq->sm, SCI_REQ_CONSTRUCTED);
 751
 752         return SCI_SUCCESS;
 753 }
 754
 755 static enum sci_status sci_io_request_construct_basic_sata(struct isci_request *ireq)
 756 {
 757         enum sci_status status;
 758         bool copy = false;
 759         struct sas_task *task = isci_request_access_task(ireq);
 760
 761         ireq->protocol = SAS_PROTOCOL_STP;
 762
 763         copy = (task->data_dir == DMA_NONE) ? false : true;
 764
 765         status = sci_io_request_construct_sata(ireq,
 766                                                 task->total_xfer_len,
 767                                                 task->data_dir,
 768                                                 copy);
 769
 770         if (status == SCI_SUCCESS)
 771                 sci_change_state(&ireq->sm, SCI_REQ_CONSTRUCTED);
 772
 773         return status;
 774 }
 775
 776 #define SCU_TASK_CONTEXT_SRAM 0x200000
 777 /**
 778  * sci_req_tx_bytes - bytes transferred when reply underruns request
 779  * @ireq: request that was terminated early
 780  */
 781 static u32 sci_req_tx_bytes(struct isci_request *ireq)
 782 {
 783         struct isci_host *ihost = ireq->owning_controller;
 784         u32 ret_val = 0;
 785
 786         if (readl(&ihost->smu_registers->address_modifier) == 0) {
 787                 void __iomem *scu_reg_base = ihost->scu_registers;
 788
 789                 /* get the bytes of data from the Address == BAR1 + 20002Ch + (256*TCi) where
 790                  *   BAR1 is the scu_registers
 791                  *   0x20002C = 0x200000 + 0x2c
 792                  *            = start of task context SRAM + offset of (type.ssp.data_offset)
 793                  *   TCi is the io_tag of struct sci_request
 794                  */
 795                 ret_val = readl(scu_reg_base +
 796                                 (SCU_TASK_CONTEXT_SRAM + offsetof(struct scu_task_context, type.ssp.data_offset)) +
 797                                 ((sizeof(struct scu_task_context)) * ISCI_TAG_TCI(ireq->io_tag)));
 798         }
 799
 800         return ret_val;
 801 }
 802
 803 enum sci_status sci_request_start(struct isci_request *ireq)
 804 {
 805         enum sci_base_request_states state;
 806         struct scu_task_context *tc = ireq->tc;
 807         struct isci_host *ihost = ireq->owning_controller;
 808
 809         state = ireq->sm.current_state_id;
 810         if (state != SCI_REQ_CONSTRUCTED) {
 811                 dev_warn(&ihost->pdev->dev,
 812                         "%s: SCIC IO Request requested to start while in wrong "
 813                          "state %d\n", __func__, state);
 814                 return SCI_FAILURE_INVALID_STATE;
 815         }
 816
 817         tc->task_index = ISCI_TAG_TCI(ireq->io_tag);
 818
 819         switch (tc->protocol_type) {
 820         case SCU_TASK_CONTEXT_PROTOCOL_SMP:
 821         case SCU_TASK_CONTEXT_PROTOCOL_SSP:
 822                 /* SSP/SMP Frame */
 823                 tc->type.ssp.tag = ireq->io_tag;
 824                 tc->type.ssp.target_port_transfer_tag = 0xFFFF;
 825                 break;
 826
 827         case SCU_TASK_CONTEXT_PROTOCOL_STP:
 828                 /* STP/SATA Frame
 829                  * tc->type.stp.ncq_tag = ireq->ncq_tag;
 830                  */
 831                 break;
 832
 833         case SCU_TASK_CONTEXT_PROTOCOL_NONE:
 834                 /* / @todo When do we set no protocol type? */
 835                 break;
 836
 837         default:
 838                 /* This should never happen since we build the IO
 839                  * requests */
 840                 break;
 841         }
 842
 843         /* Add to the post_context the io tag value */
 844         ireq->post_context |= ISCI_TAG_TCI(ireq->io_tag);
 845
 846         /* Everything is good go ahead and change state */
 847         sci_change_state(&ireq->sm, SCI_REQ_STARTED);
 848
 849         return SCI_SUCCESS;
 850 }
 851
 852 enum sci_status
 853 sci_io_request_terminate(struct isci_request *ireq)
 854 {
 855         enum sci_base_request_states state;
 856
 857         state = ireq->sm.current_state_id;
 858
 859         switch (state) {
 860         case SCI_REQ_CONSTRUCTED:
 861                 /* Set to make sure no HW terminate posting is done: */
 862                 set_bit(IREQ_TC_ABORT_POSTED, &ireq->flags);
 863                 ireq->scu_status = SCU_TASK_DONE_TASK_ABORT;
 864                 ireq->sci_status = SCI_FAILURE_IO_TERMINATED;
 865                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
 866                 return SCI_SUCCESS;
 867         case SCI_REQ_STARTED:
 868         case SCI_REQ_TASK_WAIT_TC_COMP:
 869         case SCI_REQ_SMP_WAIT_RESP:
 870         case SCI_REQ_SMP_WAIT_TC_COMP:
 871         case SCI_REQ_STP_UDMA_WAIT_TC_COMP:
 872         case SCI_REQ_STP_UDMA_WAIT_D2H:
 873         case SCI_REQ_STP_NON_DATA_WAIT_H2D:
 874         case SCI_REQ_STP_NON_DATA_WAIT_D2H:
 875         case SCI_REQ_STP_PIO_WAIT_H2D:
 876         case SCI_REQ_STP_PIO_WAIT_FRAME:
 877         case SCI_REQ_STP_PIO_DATA_IN:
 878         case SCI_REQ_STP_PIO_DATA_OUT:
 879         case SCI_REQ_ATAPI_WAIT_H2D:
 880         case SCI_REQ_ATAPI_WAIT_PIO_SETUP:
 881         case SCI_REQ_ATAPI_WAIT_D2H:
 882         case SCI_REQ_ATAPI_WAIT_TC_COMP:
 883                 /* Fall through and change state to ABORTING... */
 884         case SCI_REQ_TASK_WAIT_TC_RESP:
 885                 /* The task frame was already confirmed to have been
 886                  * sent by the SCU HW.  Since the state machine is
 887                  * now only waiting for the task response itself,
 888                  * abort the request and complete it immediately
 889                  * and don't wait for the task response.
 890                  */
 891                 sci_change_state(&ireq->sm, SCI_REQ_ABORTING);
 892                 fallthrough;    /* and handle like ABORTING */
 893         case SCI_REQ_ABORTING:
 894                 if (!isci_remote_device_is_safe_to_abort(ireq->target_device))
 895                         set_bit(IREQ_PENDING_ABORT, &ireq->flags);
 896                 else
 897                         clear_bit(IREQ_PENDING_ABORT, &ireq->flags);
 898                 /* If the request is only waiting on the remote device
 899                  * suspension, return SUCCESS so the caller will wait too.
 900                  */
 901                 return SCI_SUCCESS;
 902         case SCI_REQ_COMPLETED:
 903         default:
 904                 dev_warn(&ireq->owning_controller->pdev->dev,
 905                          "%s: SCIC IO Request requested to abort while in wrong "
 906                          "state %d\n", __func__, ireq->sm.current_state_id);
 907                 break;
 908         }
 909
 910         return SCI_FAILURE_INVALID_STATE;
 911 }
 912
 913 enum sci_status sci_request_complete(struct isci_request *ireq)
 914 {
 915         enum sci_base_request_states state;
 916         struct isci_host *ihost = ireq->owning_controller;
 917
 918         state = ireq->sm.current_state_id;
 919         if (WARN_ONCE(state != SCI_REQ_COMPLETED,
 920                       "isci: request completion from wrong state (%s)\n",
 921                       req_state_name(state)))
 922                 return SCI_FAILURE_INVALID_STATE;
 923
 924         if (ireq->saved_rx_frame_index != SCU_INVALID_FRAME_INDEX)
 925                 sci_controller_release_frame(ihost,
 926                                                   ireq->saved_rx_frame_index);
 927
 928         /* XXX can we just stop the machine and remove the 'final' state? */
 929         sci_change_state(&ireq->sm, SCI_REQ_FINAL);
 930         return SCI_SUCCESS;
 931 }
 932
 933 enum sci_status sci_io_request_event_handler(struct isci_request *ireq,
 934                                                   u32 event_code)
 935 {
 936         enum sci_base_request_states state;
 937         struct isci_host *ihost = ireq->owning_controller;
 938
 939         state = ireq->sm.current_state_id;
 940
 941         if (state != SCI_REQ_STP_PIO_DATA_IN) {
 942                 dev_warn(&ihost->pdev->dev, "%s: (%x) in wrong state %s\n",
 943                          __func__, event_code, req_state_name(state));
 944
 945                 return SCI_FAILURE_INVALID_STATE;
 946         }
 947
 948         switch (scu_get_event_specifier(event_code)) {
 949         case SCU_TASK_DONE_CRC_ERR << SCU_EVENT_SPECIFIC_CODE_SHIFT:
 950                 /* We are waiting for data and the SCU has R_ERR the data frame.
 951                  * Go back to waiting for the D2H Register FIS
 952                  */
 953                 sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
 954                 return SCI_SUCCESS;
 955         default:
 956                 dev_err(&ihost->pdev->dev,
 957                         "%s: pio request unexpected event %#x\n",
 958                         __func__, event_code);
 959
 960                 /* TODO Should we fail the PIO request when we get an
 961                  * unexpected event?
 962                  */
 963                 return SCI_FAILURE;
 964         }
 965 }
 966
 967 /*
 968  * This function copies response data for requests returning response data
 969  *    instead of sense data.
 970  * @sci_req: This parameter specifies the request object for which to copy
 971  *    the response data.
 972  */
 973 static void sci_io_request_copy_response(struct isci_request *ireq)
 974 {
 975         void *resp_buf;
 976         u32 len;
 977         struct ssp_response_iu *ssp_response;
 978         struct isci_tmf *isci_tmf = isci_request_access_tmf(ireq);
 979
 980         ssp_response = &ireq->ssp.rsp;
 981
 982         resp_buf = &isci_tmf->resp.resp_iu;
 983
 984         len = min_t(u32,
 985                     SSP_RESP_IU_MAX_SIZE,
 986                     be32_to_cpu(ssp_response->response_data_len));
 987
 988         memcpy(resp_buf, ssp_response->resp_data, len);
 989 }
 990
 991 static enum sci_status
 992 request_started_state_tc_event(struct isci_request *ireq,
 993                                u32 completion_code)
 994 {
 995         struct ssp_response_iu *resp_iu;
 996         u8 datapres;
 997
 998         /* TODO: Any SDMA return code of other than 0 is bad decode 0x003C0000
 999          * to determine SDMA status
1000          */
1001         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1002         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1003                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1004                 ireq->sci_status = SCI_SUCCESS;
1005                 break;
1006         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_EARLY_RESP): {
1007                 /* There are times when the SCU hardware will return an early
1008                  * response because the io request specified more data than is
1009                  * returned by the target device (mode pages, inquiry data,
1010                  * etc.).  We must check the response stats to see if this is
1011                  * truly a failed request or a good request that just got
1012                  * completed early.
1013                  */
1014                 struct ssp_response_iu *resp = &ireq->ssp.rsp;
1015                 ssize_t word_cnt = SSP_RESP_IU_MAX_SIZE / sizeof(u32);
1016
1017                 sci_swab32_cpy(&ireq->ssp.rsp,
1018                                &ireq->ssp.rsp,
1019                                word_cnt);
1020
1021                 if (resp->status == 0) {
1022                         ireq->scu_status = SCU_TASK_DONE_GOOD;
1023                         ireq->sci_status = SCI_SUCCESS_IO_DONE_EARLY;
1024                 } else {
1025                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1026                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
1027                 }
1028                 break;
1029         }
1030         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_CHECK_RESPONSE): {
1031                 ssize_t word_cnt = SSP_RESP_IU_MAX_SIZE / sizeof(u32);
1032
1033                 sci_swab32_cpy(&ireq->ssp.rsp,
1034                                &ireq->ssp.rsp,
1035                                word_cnt);
1036
1037                 ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1038                 ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
1039                 break;
1040         }
1041
1042         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_RESP_LEN_ERR):
1043                 /* TODO With TASK_DONE_RESP_LEN_ERR is the response frame
1044                  * guaranteed to be received before this completion status is
1045                  * posted?
1046                  */
1047                 resp_iu = &ireq->ssp.rsp;
1048                 datapres = resp_iu->datapres;
1049
1050                 if (datapres == SAS_DATAPRES_RESPONSE_DATA ||
1051                     datapres == SAS_DATAPRES_SENSE_DATA) {
1052                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1053                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
1054                 } else {
1055                         ireq->scu_status = SCU_TASK_DONE_GOOD;
1056                         ireq->sci_status = SCI_SUCCESS;
1057                 }
1058                 break;
1059         /* only stp device gets suspended. */
1060         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_ACK_NAK_TO):
1061         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LL_PERR):
1062         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_NAK_ERR):
1063         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_DATA_LEN_ERR):
1064         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LL_ABORT_ERR):
1065         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_XR_WD_LEN):
1066         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_MAX_PLD_ERR):
1067         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_RESP):
1068         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_SDBFIS):
1069         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_REG_ERR):
1070         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SDB_ERR):
1071                 if (ireq->protocol == SAS_PROTOCOL_STP) {
1072                         ireq->scu_status = SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
1073                                            SCU_COMPLETION_TL_STATUS_SHIFT;
1074                         ireq->sci_status = SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED;
1075                 } else {
1076                         ireq->scu_status = SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
1077                                            SCU_COMPLETION_TL_STATUS_SHIFT;
1078                         ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1079                 }
1080                 break;
1081
1082         /* both stp/ssp device gets suspended */
1083         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_LF_ERR):
1084         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_WRONG_DESTINATION):
1085         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1):
1086         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2):
1087         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3):
1088         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_BAD_DESTINATION):
1089         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_ZONE_VIOLATION):
1090         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_STP_RESOURCES_BUSY):
1091         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_PROTOCOL_NOT_SUPPORTED):
1092         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_OPEN_REJECT_CONNECTION_RATE_NOT_SUPPORTED):
1093                 ireq->scu_status = SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
1094                                    SCU_COMPLETION_TL_STATUS_SHIFT;
1095                 ireq->sci_status = SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED;
1096                 break;
1097
1098         /* neither ssp nor stp gets suspended. */
1099         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_NAK_CMD_ERR):
1100         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_XR):
1101         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_XR_IU_LEN_ERR):
1102         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SDMA_ERR):
1103         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_OFFSET_ERR):
1104         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_EXCESS_DATA):
1105         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_RESP_TO_ERR):
1106         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_UFI_ERR):
1107         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_FRM_TYPE_ERR):
1108         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_LL_RX_ERR):
1109         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_DATA):
1110         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_OPEN_FAIL):
1111         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_VIIT_ENTRY_NV):
1112         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_IIT_ENTRY_NV):
1113         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_RNCNV_OUTBOUND):
1114         default:
1115                 ireq->scu_status = SCU_GET_COMPLETION_TL_STATUS(completion_code) >>
1116                                    SCU_COMPLETION_TL_STATUS_SHIFT;
1117                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1118                 break;
1119         }
1120
1121         /*
1122          * TODO: This is probably wrong for ACK/NAK timeout conditions
1123          */
1124
1125         /* In all cases we will treat this as the completion of the IO req. */
1126         sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1127         return SCI_SUCCESS;
1128 }
1129
1130 static enum sci_status
1131 request_aborting_state_tc_event(struct isci_request *ireq,
1132                                 u32 completion_code)
1133 {
1134         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1135         case (SCU_TASK_DONE_GOOD << SCU_COMPLETION_TL_STATUS_SHIFT):
1136         case (SCU_TASK_DONE_TASK_ABORT << SCU_COMPLETION_TL_STATUS_SHIFT):
1137                 ireq->scu_status = SCU_TASK_DONE_TASK_ABORT;
1138                 ireq->sci_status = SCI_FAILURE_IO_TERMINATED;
1139                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1140                 break;
1141
1142         default:
1143                 /* Unless we get some strange error wait for the task abort to complete
1144                  * TODO: Should there be a state change for this completion?
1145                  */
1146                 break;
1147         }
1148
1149         return SCI_SUCCESS;
1150 }
1151
1152 static enum sci_status ssp_task_request_await_tc_event(struct isci_request *ireq,
1153                                                        u32 completion_code)
1154 {
1155         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1156         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1157                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1158                 ireq->sci_status = SCI_SUCCESS;
1159                 sci_change_state(&ireq->sm, SCI_REQ_TASK_WAIT_TC_RESP);
1160                 break;
1161         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_ACK_NAK_TO):
1162                 /* Currently, the decision is to simply allow the task request
1163                  * to timeout if the task IU wasn't received successfully.
1164                  * There is a potential for receiving multiple task responses if
1165                  * we decide to send the task IU again.
1166                  */
1167                 dev_warn(&ireq->owning_controller->pdev->dev,
1168                          "%s: TaskRequest:0x%p CompletionCode:%x - "
1169                          "ACK/NAK timeout\n", __func__, ireq,
1170                          completion_code);
1171
1172                 sci_change_state(&ireq->sm, SCI_REQ_TASK_WAIT_TC_RESP);
1173                 break;
1174         default:
1175                 /*
1176                  * All other completion status cause the IO to be complete.
1177                  * If a NAK was received, then it is up to the user to retry
1178                  * the request.
1179                  */
1180                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
1181                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1182                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1183                 break;
1184         }
1185
1186         return SCI_SUCCESS;
1187 }
1188
1189 static enum sci_status
1190 smp_request_await_response_tc_event(struct isci_request *ireq,
1191                                     u32 completion_code)
1192 {
1193         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1194         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1195                 /* In the AWAIT RESPONSE state, any TC completion is
1196                  * unexpected.  but if the TC has success status, we
1197                  * complete the IO anyway.
1198                  */
1199                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1200                 ireq->sci_status = SCI_SUCCESS;
1201                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1202                 break;
1203         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_RESP_TO_ERR):
1204         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_UFI_ERR):
1205         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_FRM_TYPE_ERR):
1206         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_SMP_LL_RX_ERR):
1207                 /* These status has been seen in a specific LSI
1208                  * expander, which sometimes is not able to send smp
1209                  * response within 2 ms. This causes our hardware break
1210                  * the connection and set TC completion with one of
1211                  * these SMP_XXX_XX_ERR status. For these type of error,
1212                  * we ask ihost user to retry the request.
1213                  */
1214                 ireq->scu_status = SCU_TASK_DONE_SMP_RESP_TO_ERR;
1215                 ireq->sci_status = SCI_FAILURE_RETRY_REQUIRED;
1216                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1217                 break;
1218         default:
1219                 /* All other completion status cause the IO to be complete.  If a NAK
1220                  * was received, then it is up to the user to retry the request
1221                  */
1222                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
1223                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1224                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1225                 break;
1226         }
1227
1228         return SCI_SUCCESS;
1229 }
1230
1231 static enum sci_status
1232 smp_request_await_tc_event(struct isci_request *ireq,
1233                            u32 completion_code)
1234 {
1235         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1236         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1237                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1238                 ireq->sci_status = SCI_SUCCESS;
1239                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1240                 break;
1241         default:
1242                 /* All other completion status cause the IO to be
1243                  * complete.  If a NAK was received, then it is up to
1244                  * the user to retry the request.
1245                  */
1246                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
1247                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1248                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1249                 break;
1250         }
1251
1252         return SCI_SUCCESS;
1253 }
1254
1255 static struct scu_sgl_element *pio_sgl_next(struct isci_stp_request *stp_req)
1256 {
1257         struct scu_sgl_element *sgl;
1258         struct scu_sgl_element_pair *sgl_pair;
1259         struct isci_request *ireq = to_ireq(stp_req);
1260         struct isci_stp_pio_sgl *pio_sgl = &stp_req->sgl;
1261
1262         sgl_pair = to_sgl_element_pair(ireq, pio_sgl->index);
1263         if (!sgl_pair)
1264                 sgl = NULL;
1265         else if (pio_sgl->set == SCU_SGL_ELEMENT_PAIR_A) {
1266                 if (sgl_pair->B.address_lower == 0 &&
1267                     sgl_pair->B.address_upper == 0) {
1268                         sgl = NULL;
1269                 } else {
1270                         pio_sgl->set = SCU_SGL_ELEMENT_PAIR_B;
1271                         sgl = &sgl_pair->B;
1272                 }
1273         } else {
1274                 if (sgl_pair->next_pair_lower == 0 &&
1275                     sgl_pair->next_pair_upper == 0) {
1276                         sgl = NULL;
1277                 } else {
1278                         pio_sgl->index++;
1279                         pio_sgl->set = SCU_SGL_ELEMENT_PAIR_A;
1280                         sgl_pair = to_sgl_element_pair(ireq, pio_sgl->index);
1281                         sgl = &sgl_pair->A;
1282                 }
1283         }
1284
1285         return sgl;
1286 }
1287
1288 static enum sci_status
1289 stp_request_non_data_await_h2d_tc_event(struct isci_request *ireq,
1290                                         u32 completion_code)
1291 {
1292         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1293         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1294                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1295                 ireq->sci_status = SCI_SUCCESS;
1296                 sci_change_state(&ireq->sm, SCI_REQ_STP_NON_DATA_WAIT_D2H);
1297                 break;
1298
1299         default:
1300                 /* All other completion status cause the IO to be
1301                  * complete.  If a NAK was received, then it is up to
1302                  * the user to retry the request.
1303                  */
1304                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
1305                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1306                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1307                 break;
1308         }
1309
1310         return SCI_SUCCESS;
1311 }
1312
1313 #define SCU_MAX_FRAME_BUFFER_SIZE  0x400  /* 1K is the maximum SCU frame data payload */
1314
1315 /* transmit DATA_FIS from (current sgl + offset) for input
1316  * parameter length. current sgl and offset is alreay stored in the IO request
1317  */
1318 static enum sci_status sci_stp_request_pio_data_out_trasmit_data_frame(
1319         struct isci_request *ireq,
1320         u32 length)
1321 {
1322         struct isci_stp_request *stp_req = &ireq->stp.req;
1323         struct scu_task_context *task_context = ireq->tc;
1324         struct scu_sgl_element_pair *sgl_pair;
1325         struct scu_sgl_element *current_sgl;
1326
1327         /* Recycle the TC and reconstruct it for sending out DATA FIS containing
1328          * for the data from current_sgl+offset for the input length
1329          */
1330         sgl_pair = to_sgl_element_pair(ireq, stp_req->sgl.index);
1331         if (stp_req->sgl.set == SCU_SGL_ELEMENT_PAIR_A)
1332                 current_sgl = &sgl_pair->A;
1333         else
1334                 current_sgl = &sgl_pair->B;
1335
1336         /* update the TC */
1337         task_context->command_iu_upper = current_sgl->address_upper;
1338         task_context->command_iu_lower = current_sgl->address_lower;
1339         task_context->transfer_length_bytes = length;
1340         task_context->type.stp.fis_type = FIS_DATA;
1341
1342         /* send the new TC out. */
1343         return sci_controller_continue_io(ireq);
1344 }
1345
1346 static enum sci_status sci_stp_request_pio_data_out_transmit_data(struct isci_request *ireq)
1347 {
1348         struct isci_stp_request *stp_req = &ireq->stp.req;
1349         struct scu_sgl_element_pair *sgl_pair;
1350         enum sci_status status = SCI_SUCCESS;
1351         struct scu_sgl_element *sgl;
1352         u32 offset;
1353         u32 len = 0;
1354
1355         offset = stp_req->sgl.offset;
1356         sgl_pair = to_sgl_element_pair(ireq, stp_req->sgl.index);
1357         if (WARN_ONCE(!sgl_pair, "%s: null sgl element", __func__))
1358                 return SCI_FAILURE;
1359
1360         if (stp_req->sgl.set == SCU_SGL_ELEMENT_PAIR_A) {
1361                 sgl = &sgl_pair->A;
1362                 len = sgl_pair->A.length - offset;
1363         } else {
1364                 sgl = &sgl_pair->B;
1365                 len = sgl_pair->B.length - offset;
1366         }
1367
1368         if (stp_req->pio_len == 0)
1369                 return SCI_SUCCESS;
1370
1371         if (stp_req->pio_len >= len) {
1372                 status = sci_stp_request_pio_data_out_trasmit_data_frame(ireq, len);
1373                 if (status != SCI_SUCCESS)
1374                         return status;
1375                 stp_req->pio_len -= len;
1376
1377                 /* update the current sgl, offset and save for future */
1378                 sgl = pio_sgl_next(stp_req);
1379                 offset = 0;
1380         } else if (stp_req->pio_len < len) {
1381                 sci_stp_request_pio_data_out_trasmit_data_frame(ireq, stp_req->pio_len);
1382
1383                 /* Sgl offset will be adjusted and saved for future */
1384                 offset += stp_req->pio_len;
1385                 sgl->address_lower += stp_req->pio_len;
1386                 stp_req->pio_len = 0;
1387         }
1388
1389         stp_req->sgl.offset = offset;
1390
1391         return status;
1392 }
1393
1394 /**
1395  * sci_stp_request_pio_data_in_copy_data_buffer()
1396  * @stp_req: The request that is used for the SGL processing.
1397  * @data_buf: The buffer of data to be copied.
1398  * @len: The length of the data transfer.
1399  *
1400  * Copy the data from the buffer for the length specified to the IO request SGL
1401  * specified data region. enum sci_status
1402  */
1403 static enum sci_status
1404 sci_stp_request_pio_data_in_copy_data_buffer(struct isci_stp_request *stp_req,
1405                                              u8 *data_buf, u32 len)
1406 {
1407         struct isci_request *ireq;
1408         u8 *src_addr;
1409         int copy_len;
1410         struct sas_task *task;
1411         struct scatterlist *sg;
1412         void *kaddr;
1413         int total_len = len;
1414
1415         ireq = to_ireq(stp_req);
1416         task = isci_request_access_task(ireq);
1417         src_addr = data_buf;
1418
1419         if (task->num_scatter > 0) {
1420                 sg = task->scatter;
1421
1422                 while (total_len > 0) {
1423                         struct page *page = sg_page(sg);
1424
1425                         copy_len = min_t(int, total_len, sg_dma_len(sg));
1426                         kaddr = kmap_atomic(page);
1427                         memcpy(kaddr + sg->offset, src_addr, copy_len);
1428                         kunmap_atomic(kaddr);
1429                         total_len -= copy_len;
1430                         src_addr += copy_len;
1431                         sg = sg_next(sg);
1432                 }
1433         } else {
1434                 BUG_ON(task->total_xfer_len < total_len);
1435                 memcpy(task->scatter, src_addr, total_len);
1436         }
1437
1438         return SCI_SUCCESS;
1439 }
1440
1441 /**
1442  * sci_stp_request_pio_data_in_copy_data()
1443  * @stp_req: The PIO DATA IN request that is to receive the data.
1444  * @data_buffer: The buffer to copy from.
1445  *
1446  * Copy the data buffer to the io request data region. enum sci_status
1447  */
1448 static enum sci_status sci_stp_request_pio_data_in_copy_data(
1449         struct isci_stp_request *stp_req,
1450         u8 *data_buffer)
1451 {
1452         enum sci_status status;
1453
1454         /*
1455          * If there is less than 1K remaining in the transfer request
1456          * copy just the data for the transfer */
1457         if (stp_req->pio_len < SCU_MAX_FRAME_BUFFER_SIZE) {
1458                 status = sci_stp_request_pio_data_in_copy_data_buffer(
1459                         stp_req, data_buffer, stp_req->pio_len);
1460
1461                 if (status == SCI_SUCCESS)
1462                         stp_req->pio_len = 0;
1463         } else {
1464                 /* We are transfering the whole frame so copy */
1465                 status = sci_stp_request_pio_data_in_copy_data_buffer(
1466                         stp_req, data_buffer, SCU_MAX_FRAME_BUFFER_SIZE);
1467
1468                 if (status == SCI_SUCCESS)
1469                         stp_req->pio_len -= SCU_MAX_FRAME_BUFFER_SIZE;
1470         }
1471
1472         return status;
1473 }
1474
1475 static enum sci_status
1476 stp_request_pio_await_h2d_completion_tc_event(struct isci_request *ireq,
1477                                               u32 completion_code)
1478 {
1479         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1480         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1481                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1482                 ireq->sci_status = SCI_SUCCESS;
1483                 sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1484                 break;
1485
1486         default:
1487                 /* All other completion status cause the IO to be
1488                  * complete.  If a NAK was received, then it is up to
1489                  * the user to retry the request.
1490                  */
1491                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
1492                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1493                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1494                 break;
1495         }
1496
1497         return SCI_SUCCESS;
1498 }
1499
1500 static enum sci_status
1501 pio_data_out_tx_done_tc_event(struct isci_request *ireq,
1502                               u32 completion_code)
1503 {
1504         enum sci_status status = SCI_SUCCESS;
1505         bool all_frames_transferred = false;
1506         struct isci_stp_request *stp_req = &ireq->stp.req;
1507
1508         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
1509         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
1510                 /* Transmit data */
1511                 if (stp_req->pio_len != 0) {
1512                         status = sci_stp_request_pio_data_out_transmit_data(ireq);
1513                         if (status == SCI_SUCCESS) {
1514                                 if (stp_req->pio_len == 0)
1515                                         all_frames_transferred = true;
1516                         }
1517                 } else if (stp_req->pio_len == 0) {
1518                         /*
1519                          * this will happen if the all data is written at the
1520                          * first time after the pio setup fis is received
1521                          */
1522                         all_frames_transferred  = true;
1523                 }
1524
1525                 /* all data transferred. */
1526                 if (all_frames_transferred) {
1527                         /*
1528                          * Change the state to SCI_REQ_STP_PIO_DATA_IN
1529                          * and wait for PIO_SETUP fis / or D2H REg fis. */
1530                         sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1531                 }
1532                 break;
1533
1534         default:
1535                 /*
1536                  * All other completion status cause the IO to be complete.
1537                  * If a NAK was received, then it is up to the user to retry
1538                  * the request.
1539                  */
1540                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
1541                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1542                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1543                 break;
1544         }
1545
1546         return status;
1547 }
1548
1549 static enum sci_status sci_stp_request_udma_general_frame_handler(struct isci_request *ireq,
1550                                                                        u32 frame_index)
1551 {
1552         struct isci_host *ihost = ireq->owning_controller;
1553         struct dev_to_host_fis *frame_header;
1554         enum sci_status status;
1555         u32 *frame_buffer;
1556
1557         status = sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1558                                                                frame_index,
1559                                                                (void **)&frame_header);
1560
1561         if ((status == SCI_SUCCESS) &&
1562             (frame_header->fis_type == FIS_REGD2H)) {
1563                 sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1564                                                               frame_index,
1565                                                               (void **)&frame_buffer);
1566
1567                 sci_controller_copy_sata_response(&ireq->stp.rsp,
1568                                                        frame_header,
1569                                                        frame_buffer);
1570         }
1571
1572         sci_controller_release_frame(ihost, frame_index);
1573
1574         return status;
1575 }
1576
1577 static enum sci_status process_unsolicited_fis(struct isci_request *ireq,
1578                                                u32 frame_index)
1579 {
1580         struct isci_host *ihost = ireq->owning_controller;
1581         enum sci_status status;
1582         struct dev_to_host_fis *frame_header;
1583         u32 *frame_buffer;
1584
1585         status = sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1586                                                           frame_index,
1587                                                           (void **)&frame_header);
1588
1589         if (status != SCI_SUCCESS)
1590                 return status;
1591
1592         if (frame_header->fis_type != FIS_REGD2H) {
1593                 dev_err(&ireq->isci_host->pdev->dev,
1594                         "%s ERROR: invalid fis type 0x%X\n",
1595                         __func__, frame_header->fis_type);
1596                 return SCI_FAILURE;
1597         }
1598
1599         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1600                                                  frame_index,
1601                                                  (void **)&frame_buffer);
1602
1603         sci_controller_copy_sata_response(&ireq->stp.rsp,
1604                                           (u32 *)frame_header,
1605                                           frame_buffer);
1606
1607         /* Frame has been decoded return it to the controller */
1608         sci_controller_release_frame(ihost, frame_index);
1609
1610         return status;
1611 }
1612
1613 static enum sci_status atapi_d2h_reg_frame_handler(struct isci_request *ireq,
1614                                                    u32 frame_index)
1615 {
1616         struct sas_task *task = isci_request_access_task(ireq);
1617         enum sci_status status;
1618
1619         status = process_unsolicited_fis(ireq, frame_index);
1620
1621         if (status == SCI_SUCCESS) {
1622                 if (ireq->stp.rsp.status & ATA_ERR)
1623                         status = SCI_FAILURE_IO_RESPONSE_VALID;
1624         } else {
1625                 status = SCI_FAILURE_IO_RESPONSE_VALID;
1626         }
1627
1628         if (status != SCI_SUCCESS) {
1629                 ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1630                 ireq->sci_status = status;
1631         } else {
1632                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1633                 ireq->sci_status = SCI_SUCCESS;
1634         }
1635
1636         /* the d2h ufi is the end of non-data commands */
1637         if (task->data_dir == DMA_NONE)
1638                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1639
1640         return status;
1641 }
1642
1643 static void scu_atapi_reconstruct_raw_frame_task_context(struct isci_request *ireq)
1644 {
1645         struct ata_device *dev = sas_to_ata_dev(ireq->target_device->domain_dev);
1646         void *atapi_cdb = ireq->ttype_ptr.io_task_ptr->ata_task.atapi_packet;
1647         struct scu_task_context *task_context = ireq->tc;
1648
1649         /* fill in the SCU Task Context for a DATA fis containing CDB in Raw Frame
1650          * type. The TC for previous Packet fis was already there, we only need to
1651          * change the H2D fis content.
1652          */
1653         memset(&ireq->stp.cmd, 0, sizeof(struct host_to_dev_fis));
1654         memcpy(((u8 *)&ireq->stp.cmd + sizeof(u32)), atapi_cdb, ATAPI_CDB_LEN);
1655         memset(&(task_context->type.stp), 0, sizeof(struct stp_task_context));
1656         task_context->type.stp.fis_type = FIS_DATA;
1657         task_context->transfer_length_bytes = dev->cdb_len;
1658 }
1659
1660 static void scu_atapi_construct_task_context(struct isci_request *ireq)
1661 {
1662         struct ata_device *dev = sas_to_ata_dev(ireq->target_device->domain_dev);
1663         struct sas_task *task = isci_request_access_task(ireq);
1664         struct scu_task_context *task_context = ireq->tc;
1665         int cdb_len = dev->cdb_len;
1666
1667         /* reference: SSTL 1.13.4.2
1668          * task_type, sata_direction
1669          */
1670         if (task->data_dir == DMA_TO_DEVICE) {
1671                 task_context->task_type = SCU_TASK_TYPE_PACKET_DMA_OUT;
1672                 task_context->sata_direction = 0;
1673         } else {
1674                 /* todo: for NO_DATA command, we need to send out raw frame. */
1675                 task_context->task_type = SCU_TASK_TYPE_PACKET_DMA_IN;
1676                 task_context->sata_direction = 1;
1677         }
1678
1679         memset(&task_context->type.stp, 0, sizeof(task_context->type.stp));
1680         task_context->type.stp.fis_type = FIS_DATA;
1681
1682         memset(&ireq->stp.cmd, 0, sizeof(ireq->stp.cmd));
1683         memcpy(&ireq->stp.cmd.lbal, task->ata_task.atapi_packet, cdb_len);
1684         task_context->ssp_command_iu_length = cdb_len / sizeof(u32);
1685
1686         /* task phase is set to TX_CMD */
1687         task_context->task_phase = 0x1;
1688
1689         /* retry counter */
1690         task_context->stp_retry_count = 0;
1691
1692         /* data transfer size. */
1693         task_context->transfer_length_bytes = task->total_xfer_len;
1694
1695         /* setup sgl */
1696         sci_request_build_sgl(ireq);
1697 }
1698
1699 enum sci_status
1700 sci_io_request_frame_handler(struct isci_request *ireq,
1701                                   u32 frame_index)
1702 {
1703         struct isci_host *ihost = ireq->owning_controller;
1704         struct isci_stp_request *stp_req = &ireq->stp.req;
1705         enum sci_base_request_states state;
1706         enum sci_status status;
1707         ssize_t word_cnt;
1708
1709         state = ireq->sm.current_state_id;
1710         switch (state)  {
1711         case SCI_REQ_STARTED: {
1712                 struct ssp_frame_hdr ssp_hdr;
1713                 void *frame_header;
1714
1715                 sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1716                                                               frame_index,
1717                                                               &frame_header);
1718
1719                 word_cnt = sizeof(struct ssp_frame_hdr) / sizeof(u32);
1720                 sci_swab32_cpy(&ssp_hdr, frame_header, word_cnt);
1721
1722                 if (ssp_hdr.frame_type == SSP_RESPONSE) {
1723                         struct ssp_response_iu *resp_iu;
1724                         ssize_t word_cnt = SSP_RESP_IU_MAX_SIZE / sizeof(u32);
1725
1726                         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1727                                                                       frame_index,
1728                                                                       (void **)&resp_iu);
1729
1730                         sci_swab32_cpy(&ireq->ssp.rsp, resp_iu, word_cnt);
1731
1732                         resp_iu = &ireq->ssp.rsp;
1733
1734                         if (resp_iu->datapres == SAS_DATAPRES_RESPONSE_DATA ||
1735                             resp_iu->datapres == SAS_DATAPRES_SENSE_DATA) {
1736                                 ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1737                                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1738                         } else {
1739                                 ireq->scu_status = SCU_TASK_DONE_GOOD;
1740                                 ireq->sci_status = SCI_SUCCESS;
1741                         }
1742                 } else {
1743                         /* not a response frame, why did it get forwarded? */
1744                         dev_err(&ihost->pdev->dev,
1745                                 "%s: SCIC IO Request 0x%p received unexpected "
1746                                 "frame %d type 0x%02x\n", __func__, ireq,
1747                                 frame_index, ssp_hdr.frame_type);
1748                 }
1749
1750                 /*
1751                  * In any case we are done with this frame buffer return it to
1752                  * the controller
1753                  */
1754                 sci_controller_release_frame(ihost, frame_index);
1755
1756                 return SCI_SUCCESS;
1757         }
1758
1759         case SCI_REQ_TASK_WAIT_TC_RESP:
1760                 sci_io_request_copy_response(ireq);
1761                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1762                 sci_controller_release_frame(ihost, frame_index);
1763                 return SCI_SUCCESS;
1764
1765         case SCI_REQ_SMP_WAIT_RESP: {
1766                 struct sas_task *task = isci_request_access_task(ireq);
1767                 struct scatterlist *sg = &task->smp_task.smp_resp;
1768                 void *frame_header, *kaddr;
1769                 u8 *rsp;
1770
1771                 sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1772                                                          frame_index,
1773                                                          &frame_header);
1774                 kaddr = kmap_atomic(sg_page(sg));
1775                 rsp = kaddr + sg->offset;
1776                 sci_swab32_cpy(rsp, frame_header, 1);
1777
1778                 if (rsp[0] == SMP_RESPONSE) {
1779                         void *smp_resp;
1780
1781                         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1782                                                                  frame_index,
1783                                                                  &smp_resp);
1784
1785                         word_cnt = (sg->length/4)-1;
1786                         if (word_cnt > 0)
1787                                 word_cnt = min_t(unsigned int, word_cnt,
1788                                                  SCU_UNSOLICITED_FRAME_BUFFER_SIZE/4);
1789                         sci_swab32_cpy(rsp + 4, smp_resp, word_cnt);
1790
1791                         ireq->scu_status = SCU_TASK_DONE_GOOD;
1792                         ireq->sci_status = SCI_SUCCESS;
1793                         sci_change_state(&ireq->sm, SCI_REQ_SMP_WAIT_TC_COMP);
1794                 } else {
1795                         /*
1796                          * This was not a response frame why did it get
1797                          * forwarded?
1798                          */
1799                         dev_err(&ihost->pdev->dev,
1800                                 "%s: SCIC SMP Request 0x%p received unexpected "
1801                                 "frame %d type 0x%02x\n",
1802                                 __func__,
1803                                 ireq,
1804                                 frame_index,
1805                                 rsp[0]);
1806
1807                         ireq->scu_status = SCU_TASK_DONE_SMP_FRM_TYPE_ERR;
1808                         ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
1809                         sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1810                 }
1811                 kunmap_atomic(kaddr);
1812
1813                 sci_controller_release_frame(ihost, frame_index);
1814
1815                 return SCI_SUCCESS;
1816         }
1817
1818         case SCI_REQ_STP_UDMA_WAIT_TC_COMP:
1819                 return sci_stp_request_udma_general_frame_handler(ireq,
1820                                                                        frame_index);
1821
1822         case SCI_REQ_STP_UDMA_WAIT_D2H:
1823                 /* Use the general frame handler to copy the resposne data */
1824                 status = sci_stp_request_udma_general_frame_handler(ireq, frame_index);
1825
1826                 if (status != SCI_SUCCESS)
1827                         return status;
1828
1829                 ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1830                 ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
1831                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1832                 return SCI_SUCCESS;
1833
1834         case SCI_REQ_STP_NON_DATA_WAIT_D2H: {
1835                 struct dev_to_host_fis *frame_header;
1836                 u32 *frame_buffer;
1837
1838                 status = sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1839                                                                        frame_index,
1840                                                                        (void **)&frame_header);
1841
1842                 if (status != SCI_SUCCESS) {
1843                         dev_err(&ihost->pdev->dev,
1844                                 "%s: SCIC IO Request 0x%p could not get frame "
1845                                 "header for frame index %d, status %x\n",
1846                                 __func__,
1847                                 stp_req,
1848                                 frame_index,
1849                                 status);
1850
1851                         return status;
1852                 }
1853
1854                 switch (frame_header->fis_type) {
1855                 case FIS_REGD2H:
1856                         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1857                                                                       frame_index,
1858                                                                       (void **)&frame_buffer);
1859
1860                         sci_controller_copy_sata_response(&ireq->stp.rsp,
1861                                                                frame_header,
1862                                                                frame_buffer);
1863
1864                         /* The command has completed with error */
1865                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1866                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
1867                         break;
1868
1869                 default:
1870                         dev_warn(&ihost->pdev->dev,
1871                                  "%s: IO Request:0x%p Frame Id:%d protocol "
1872                                   "violation occurred\n", __func__, stp_req,
1873                                   frame_index);
1874
1875                         ireq->scu_status = SCU_TASK_DONE_UNEXP_FIS;
1876                         ireq->sci_status = SCI_FAILURE_PROTOCOL_VIOLATION;
1877                         break;
1878                 }
1879
1880                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1881
1882                 /* Frame has been decoded return it to the controller */
1883                 sci_controller_release_frame(ihost, frame_index);
1884
1885                 return status;
1886         }
1887
1888         case SCI_REQ_STP_PIO_WAIT_FRAME: {
1889                 struct sas_task *task = isci_request_access_task(ireq);
1890                 struct dev_to_host_fis *frame_header;
1891                 u32 *frame_buffer;
1892
1893                 status = sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1894                                                                        frame_index,
1895                                                                        (void **)&frame_header);
1896
1897                 if (status != SCI_SUCCESS) {
1898                         dev_err(&ihost->pdev->dev,
1899                                 "%s: SCIC IO Request 0x%p could not get frame "
1900                                 "header for frame index %d, status %x\n",
1901                                 __func__, stp_req, frame_index, status);
1902                         return status;
1903                 }
1904
1905                 switch (frame_header->fis_type) {
1906                 case FIS_PIO_SETUP:
1907                         /* Get from the frame buffer the PIO Setup Data */
1908                         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1909                                                                       frame_index,
1910                                                                       (void **)&frame_buffer);
1911
1912                         /* Get the data from the PIO Setup The SCU Hardware
1913                          * returns first word in the frame_header and the rest
1914                          * of the data is in the frame buffer so we need to
1915                          * back up one dword
1916                          */
1917
1918                         /* transfer_count: first 16bits in the 4th dword */
1919                         stp_req->pio_len = frame_buffer[3] & 0xffff;
1920
1921                         /* status: 4th byte in the 3rd dword */
1922                         stp_req->status = (frame_buffer[2] >> 24) & 0xff;
1923
1924                         sci_controller_copy_sata_response(&ireq->stp.rsp,
1925                                                                frame_header,
1926                                                                frame_buffer);
1927
1928                         ireq->stp.rsp.status = stp_req->status;
1929
1930                         /* The next state is dependent on whether the
1931                          * request was PIO Data-in or Data out
1932                          */
1933                         if (task->data_dir == DMA_FROM_DEVICE) {
1934                                 sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_DATA_IN);
1935                         } else if (task->data_dir == DMA_TO_DEVICE) {
1936                                 /* Transmit data */
1937                                 status = sci_stp_request_pio_data_out_transmit_data(ireq);
1938                                 if (status != SCI_SUCCESS)
1939                                         break;
1940                                 sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_DATA_OUT);
1941                         }
1942                         break;
1943
1944                 case FIS_SETDEVBITS:
1945                         sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
1946                         break;
1947
1948                 case FIS_REGD2H:
1949                         if (frame_header->status & ATA_BUSY) {
1950                                 /*
1951                                  * Now why is the drive sending a D2H Register
1952                                  * FIS when it is still busy?  Do nothing since
1953                                  * we are still in the right state.
1954                                  */
1955                                 dev_dbg(&ihost->pdev->dev,
1956                                         "%s: SCIC PIO Request 0x%p received "
1957                                         "D2H Register FIS with BSY status "
1958                                         "0x%x\n",
1959                                         __func__,
1960                                         stp_req,
1961                                         frame_header->status);
1962                                 break;
1963                         }
1964
1965                         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
1966                                                                       frame_index,
1967                                                                       (void **)&frame_buffer);
1968
1969                         sci_controller_copy_sata_response(&ireq->stp.rsp,
1970                                                                frame_header,
1971                                                                frame_buffer);
1972
1973                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
1974                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
1975                         sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
1976                         break;
1977
1978                 default:
1979                         /* FIXME: what do we do here? */
1980                         break;
1981                 }
1982
1983                 /* Frame is decoded return it to the controller */
1984                 sci_controller_release_frame(ihost, frame_index);
1985
1986                 return status;
1987         }
1988
1989         case SCI_REQ_STP_PIO_DATA_IN: {
1990                 struct dev_to_host_fis *frame_header;
1991                 struct sata_fis_data *frame_buffer;
1992
1993                 status = sci_unsolicited_frame_control_get_header(&ihost->uf_control,
1994                                                                        frame_index,
1995                                                                        (void **)&frame_header);
1996
1997                 if (status != SCI_SUCCESS) {
1998                         dev_err(&ihost->pdev->dev,
1999                                 "%s: SCIC IO Request 0x%p could not get frame "
2000                                 "header for frame index %d, status %x\n",
2001                                 __func__,
2002                                 stp_req,
2003                                 frame_index,
2004                                 status);
2005                         return status;
2006                 }
2007
2008                 if (frame_header->fis_type != FIS_DATA) {
2009                         dev_err(&ihost->pdev->dev,
2010                                 "%s: SCIC PIO Request 0x%p received frame %d "
2011                                 "with fis type 0x%02x when expecting a data "
2012                                 "fis.\n",
2013                                 __func__,
2014                                 stp_req,
2015                                 frame_index,
2016                                 frame_header->fis_type);
2017
2018                         ireq->scu_status = SCU_TASK_DONE_GOOD;
2019                         ireq->sci_status = SCI_FAILURE_IO_REQUIRES_SCSI_ABORT;
2020                         sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2021
2022                         /* Frame is decoded return it to the controller */
2023                         sci_controller_release_frame(ihost, frame_index);
2024                         return status;
2025                 }
2026
2027                 if (stp_req->sgl.index < 0) {
2028                         ireq->saved_rx_frame_index = frame_index;
2029                         stp_req->pio_len = 0;
2030                 } else {
2031                         sci_unsolicited_frame_control_get_buffer(&ihost->uf_control,
2032                                                                       frame_index,
2033                                                                       (void **)&frame_buffer);
2034
2035                         status = sci_stp_request_pio_data_in_copy_data(stp_req,
2036                                                                             (u8 *)frame_buffer);
2037
2038                         /* Frame is decoded return it to the controller */
2039                         sci_controller_release_frame(ihost, frame_index);
2040                 }
2041
2042                 /* Check for the end of the transfer, are there more
2043                  * bytes remaining for this data transfer
2044                  */
2045                 if (status != SCI_SUCCESS || stp_req->pio_len != 0)
2046                         return status;
2047
2048                 if ((stp_req->status & ATA_BUSY) == 0) {
2049                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
2050                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
2051                         sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2052                 } else {
2053                         sci_change_state(&ireq->sm, SCI_REQ_STP_PIO_WAIT_FRAME);
2054                 }
2055                 return status;
2056         }
2057
2058         case SCI_REQ_ATAPI_WAIT_PIO_SETUP: {
2059                 struct sas_task *task = isci_request_access_task(ireq);
2060
2061                 sci_controller_release_frame(ihost, frame_index);
2062                 ireq->target_device->working_request = ireq;
2063                 if (task->data_dir == DMA_NONE) {
2064                         sci_change_state(&ireq->sm, SCI_REQ_ATAPI_WAIT_TC_COMP);
2065                         scu_atapi_reconstruct_raw_frame_task_context(ireq);
2066                 } else {
2067                         sci_change_state(&ireq->sm, SCI_REQ_ATAPI_WAIT_D2H);
2068                         scu_atapi_construct_task_context(ireq);
2069                 }
2070
2071                 sci_controller_continue_io(ireq);
2072                 return SCI_SUCCESS;
2073         }
2074         case SCI_REQ_ATAPI_WAIT_D2H:
2075                 return atapi_d2h_reg_frame_handler(ireq, frame_index);
2076         case SCI_REQ_ABORTING:
2077                 /*
2078                  * TODO: Is it even possible to get an unsolicited frame in the
2079                  * aborting state?
2080                  */
2081                 sci_controller_release_frame(ihost, frame_index);
2082                 return SCI_SUCCESS;
2083
2084         default:
2085                 dev_warn(&ihost->pdev->dev,
2086                          "%s: SCIC IO Request given unexpected frame %x while "
2087                          "in state %d\n",
2088                          __func__,
2089                          frame_index,
2090                          state);
2091
2092                 sci_controller_release_frame(ihost, frame_index);
2093                 return SCI_FAILURE_INVALID_STATE;
2094         }
2095 }
2096
2097 static enum sci_status stp_request_udma_await_tc_event(struct isci_request *ireq,
2098                                                        u32 completion_code)
2099 {
2100         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
2101         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
2102                 ireq->scu_status = SCU_TASK_DONE_GOOD;
2103                 ireq->sci_status = SCI_SUCCESS;
2104                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2105                 break;
2106         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_UNEXP_FIS):
2107         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_REG_ERR):
2108                 /* We must check ther response buffer to see if the D2H
2109                  * Register FIS was received before we got the TC
2110                  * completion.
2111                  */
2112                 if (ireq->stp.rsp.fis_type == FIS_REGD2H) {
2113                         sci_remote_device_suspend(ireq->target_device,
2114                                                   SCI_SW_SUSPEND_NORMAL);
2115
2116                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
2117                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
2118                         sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2119                 } else {
2120                         /* If we have an error completion status for the
2121                          * TC then we can expect a D2H register FIS from
2122                          * the device so we must change state to wait
2123                          * for it
2124                          */
2125                         sci_change_state(&ireq->sm, SCI_REQ_STP_UDMA_WAIT_D2H);
2126                 }
2127                 break;
2128
2129         /* TODO Check to see if any of these completion status need to
2130          * wait for the device to host register fis.
2131          */
2132         /* TODO We can retry the command for SCU_TASK_DONE_CMD_LL_R_ERR
2133          * - this comes only for B0
2134          */
2135         default:
2136                 /* All other completion status cause the IO to be complete. */
2137                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
2138                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
2139                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2140                 break;
2141         }
2142
2143         return SCI_SUCCESS;
2144 }
2145
2146 static enum sci_status atapi_raw_completion(struct isci_request *ireq, u32 completion_code,
2147                                                   enum sci_base_request_states next)
2148 {
2149         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
2150         case SCU_MAKE_COMPLETION_STATUS(SCU_TASK_DONE_GOOD):
2151                 ireq->scu_status = SCU_TASK_DONE_GOOD;
2152                 ireq->sci_status = SCI_SUCCESS;
2153                 sci_change_state(&ireq->sm, next);
2154                 break;
2155         default:
2156                 /* All other completion status cause the IO to be complete.
2157                  * If a NAK was received, then it is up to the user to retry
2158                  * the request.
2159                  */
2160                 ireq->scu_status = SCU_NORMALIZE_COMPLETION_STATUS(completion_code);
2161                 ireq->sci_status = SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR;
2162
2163                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2164                 break;
2165         }
2166
2167         return SCI_SUCCESS;
2168 }
2169
2170 static enum sci_status atapi_data_tc_completion_handler(struct isci_request *ireq,
2171                                                         u32 completion_code)
2172 {
2173         struct isci_remote_device *idev = ireq->target_device;
2174         struct dev_to_host_fis *d2h = &ireq->stp.rsp;
2175         enum sci_status status = SCI_SUCCESS;
2176
2177         switch (SCU_GET_COMPLETION_TL_STATUS(completion_code)) {
2178         case (SCU_TASK_DONE_GOOD << SCU_COMPLETION_TL_STATUS_SHIFT):
2179                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2180                 break;
2181
2182         case (SCU_TASK_DONE_UNEXP_FIS << SCU_COMPLETION_TL_STATUS_SHIFT): {
2183                 u16 len = sci_req_tx_bytes(ireq);
2184
2185                 /* likely non-error data underrun, workaround missing
2186                  * d2h frame from the controller
2187                  */
2188                 if (d2h->fis_type != FIS_REGD2H) {
2189                         d2h->fis_type = FIS_REGD2H;
2190                         d2h->flags = (1 << 6);
2191                         d2h->status = 0x50;
2192                         d2h->error = 0;
2193                         d2h->lbal = 0;
2194                         d2h->byte_count_low = len & 0xff;
2195                         d2h->byte_count_high = len >> 8;
2196                         d2h->device = 0xa0;
2197                         d2h->lbal_exp = 0;
2198                         d2h->lbam_exp = 0;
2199                         d2h->lbah_exp = 0;
2200                         d2h->_r_a = 0;
2201                         d2h->sector_count = 0x3;
2202                         d2h->sector_count_exp = 0;
2203                         d2h->_r_b = 0;
2204                         d2h->_r_c = 0;
2205                         d2h->_r_d = 0;
2206                 }
2207
2208                 ireq->scu_status = SCU_TASK_DONE_GOOD;
2209                 ireq->sci_status = SCI_SUCCESS_IO_DONE_EARLY;
2210                 status = ireq->sci_status;
2211
2212                 /* the hw will have suspended the rnc, so complete the
2213                  * request upon pending resume
2214                  */
2215                 sci_change_state(&idev->sm, SCI_STP_DEV_ATAPI_ERROR);
2216                 break;
2217         }
2218         case (SCU_TASK_DONE_EXCESS_DATA << SCU_COMPLETION_TL_STATUS_SHIFT):
2219                 /* In this case, there is no UF coming after.
2220                  * compelte the IO now.
2221                  */
2222                 ireq->scu_status = SCU_TASK_DONE_GOOD;
2223                 ireq->sci_status = SCI_SUCCESS;
2224                 sci_change_state(&ireq->sm, SCI_REQ_COMPLETED);
2225                 break;
2226
2227         default:
2228                 if (d2h->fis_type == FIS_REGD2H) {
2229                         /* UF received change the device state to ATAPI_ERROR */
2230                         status = ireq->sci_status;
2231                         sci_change_state(&idev->sm, SCI_STP_DEV_ATAPI_ERROR);
2232                 } else {
2233                         /* If receiving any non-success TC status, no UF
2234                          * received yet, then an UF for the status fis
2235                          * is coming after (XXX: suspect this is
2236                          * actually a protocol error or a bug like the
2237                          * DONE_UNEXP_FIS case)
2238                          */
2239                         ireq->scu_status = SCU_TASK_DONE_CHECK_RESPONSE;
2240                         ireq->sci_status = SCI_FAILURE_IO_RESPONSE_VALID;
2241
2242                         sci_change_state(&ireq->sm, SCI_REQ_ATAPI_WAIT_D2H);
2243                 }
2244                 break;
2245         }
2246
2247         return status;
2248 }
2249
2250 static int sci_request_smp_completion_status_is_tx_suspend(
2251         unsigned int completion_status)
2252 {
2253         switch (completion_status) {
2254         case SCU_TASK_OPEN_REJECT_WRONG_DESTINATION:
2255         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1:
2256         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2:
2257         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3:
2258         case SCU_TASK_OPEN_REJECT_BAD_DESTINATION:
2259         case SCU_TASK_OPEN_REJECT_ZONE_VIOLATION:
2260                 return 1;
2261         }
2262         return 0;
2263 }
2264
2265 static int sci_request_smp_completion_status_is_tx_rx_suspend(
2266         unsigned int completion_status)
2267 {
2268         return 0; /* There are no Tx/Rx SMP suspend conditions. */
2269 }
2270
2271 static int sci_request_ssp_completion_status_is_tx_suspend(
2272         unsigned int completion_status)
2273 {
2274         switch (completion_status) {
2275         case SCU_TASK_DONE_TX_RAW_CMD_ERR:
2276         case SCU_TASK_DONE_LF_ERR:
2277         case SCU_TASK_OPEN_REJECT_WRONG_DESTINATION:
2278         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1:
2279         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2:
2280         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3:
2281         case SCU_TASK_OPEN_REJECT_BAD_DESTINATION:
2282         case SCU_TASK_OPEN_REJECT_ZONE_VIOLATION:
2283         case SCU_TASK_OPEN_REJECT_STP_RESOURCES_BUSY:
2284         case SCU_TASK_OPEN_REJECT_PROTOCOL_NOT_SUPPORTED:
2285         case SCU_TASK_OPEN_REJECT_CONNECTION_RATE_NOT_SUPPORTED:
2286                 return 1;
2287         }
2288         return 0;
2289 }
2290
2291 static int sci_request_ssp_completion_status_is_tx_rx_suspend(
2292         unsigned int completion_status)
2293 {
2294         return 0; /* There are no Tx/Rx SSP suspend conditions. */
2295 }
2296
2297 static int sci_request_stpsata_completion_status_is_tx_suspend(
2298         unsigned int completion_status)
2299 {
2300         switch (completion_status) {
2301         case SCU_TASK_DONE_TX_RAW_CMD_ERR:
2302         case SCU_TASK_DONE_LL_R_ERR:
2303         case SCU_TASK_DONE_LL_PERR:
2304         case SCU_TASK_DONE_REG_ERR:
2305         case SCU_TASK_DONE_SDB_ERR:
2306         case SCU_TASK_OPEN_REJECT_WRONG_DESTINATION:
2307         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1:
2308         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2:
2309         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3:
2310         case SCU_TASK_OPEN_REJECT_BAD_DESTINATION:
2311         case SCU_TASK_OPEN_REJECT_ZONE_VIOLATION:
2312         case SCU_TASK_OPEN_REJECT_STP_RESOURCES_BUSY:
2313         case SCU_TASK_OPEN_REJECT_PROTOCOL_NOT_SUPPORTED:
2314         case SCU_TASK_OPEN_REJECT_CONNECTION_RATE_NOT_SUPPORTED:
2315                 return 1;
2316         }
2317         return 0;
2318 }
2319
2320
2321 static int sci_request_stpsata_completion_status_is_tx_rx_suspend(
2322         unsigned int completion_status)
2323 {
2324         switch (completion_status) {
2325         case SCU_TASK_DONE_LF_ERR:
2326         case SCU_TASK_DONE_LL_SY_TERM:
2327         case SCU_TASK_DONE_LL_LF_TERM:
2328         case SCU_TASK_DONE_BREAK_RCVD:
2329         case SCU_TASK_DONE_INV_FIS_LEN:
2330         case SCU_TASK_DONE_UNEXP_FIS:
2331         case SCU_TASK_DONE_UNEXP_SDBFIS:
2332         case SCU_TASK_DONE_MAX_PLD_ERR:
2333                 return 1;
2334         }
2335         return 0;
2336 }
2337
2338 static void sci_request_handle_suspending_completions(
2339         struct isci_request *ireq,
2340         u32 completion_code)
2341 {
2342         int is_tx = 0;
2343         int is_tx_rx = 0;
2344
2345         switch (ireq->protocol) {
2346         case SAS_PROTOCOL_SMP:
2347                 is_tx = sci_request_smp_completion_status_is_tx_suspend(
2348                         completion_code);
2349                 is_tx_rx = sci_request_smp_completion_status_is_tx_rx_suspend(
2350                         completion_code);
2351                 break;
2352         case SAS_PROTOCOL_SSP:
2353                 is_tx = sci_request_ssp_completion_status_is_tx_suspend(
2354                         completion_code);
2355                 is_tx_rx = sci_request_ssp_completion_status_is_tx_rx_suspend(
2356                         completion_code);
2357                 break;
2358         case SAS_PROTOCOL_STP:
2359                 is_tx = sci_request_stpsata_completion_status_is_tx_suspend(
2360                         completion_code);
2361                 is_tx_rx =
2362                         sci_request_stpsata_completion_status_is_tx_rx_suspend(
2363                                 completion_code);
2364                 break;
2365         default:
2366                 dev_warn(&ireq->isci_host->pdev->dev,
2367                          "%s: request %p has no valid protocol\n",
2368                          __func__, ireq);
2369                 break;
2370         }
2371         if (is_tx || is_tx_rx) {
2372                 BUG_ON(is_tx && is_tx_rx);
2373
2374                 sci_remote_node_context_suspend(
2375                         &ireq->target_device->rnc,
2376                         SCI_HW_SUSPEND,
2377                         (is_tx_rx) ? SCU_EVENT_TL_RNC_SUSPEND_TX_RX
2378                                    : SCU_EVENT_TL_RNC_SUSPEND_TX);
2379         }
2380 }
2381
2382 enum sci_status
2383 sci_io_request_tc_completion(struct isci_request *ireq,
2384                              u32 completion_code)
2385 {
2386         enum sci_base_request_states state;
2387         struct isci_host *ihost = ireq->owning_controller;
2388
2389         state = ireq->sm.current_state_id;
2390
2391         /* Decode those completions that signal upcoming suspension events. */
2392         sci_request_handle_suspending_completions(
2393                 ireq, SCU_GET_COMPLETION_TL_STATUS(completion_code));
2394
2395         switch (state) {
2396         case SCI_REQ_STARTED:
2397                 return request_started_state_tc_event(ireq, completion_code);
2398
2399         case SCI_REQ_TASK_WAIT_TC_COMP:
2400                 return ssp_task_request_await_tc_event(ireq,
2401                                                        completion_code);
2402
2403         case SCI_REQ_SMP_WAIT_RESP:
2404                 return smp_request_await_response_tc_event(ireq,
2405                                                            completion_code);
2406
2407         case SCI_REQ_SMP_WAIT_TC_COMP:
2408                 return smp_request_await_tc_event(ireq, completion_code);
2409
2410         case SCI_REQ_STP_UDMA_WAIT_TC_COMP:
2411                 return stp_request_udma_await_tc_event(ireq,
2412                                                        completion_code);
2413
2414         case SCI_REQ_STP_NON_DATA_WAIT_H2D:
2415                 return stp_request_non_data_await_h2d_tc_event(ireq,
2416                                                                completion_code);
2417
2418         case SCI_REQ_STP_PIO_WAIT_H2D:
2419                 return stp_request_pio_await_h2d_completion_tc_event(ireq,
2420                                                                      completion_code);
2421
2422         case SCI_REQ_STP_PIO_DATA_OUT:
2423                 return pio_data_out_tx_done_tc_event(ireq, completion_code);
2424
2425         case SCI_REQ_ABORTING:
2426                 return request_aborting_state_tc_event(ireq,
2427                                                        completion_code);
2428
2429         case SCI_REQ_ATAPI_WAIT_H2D:
2430                 return atapi_raw_completion(ireq, completion_code,
2431                                             SCI_REQ_ATAPI_WAIT_PIO_SETUP);
2432
2433         case SCI_REQ_ATAPI_WAIT_TC_COMP:
2434                 return atapi_raw_completion(ireq, completion_code,
2435                                             SCI_REQ_ATAPI_WAIT_D2H);
2436
2437         case SCI_REQ_ATAPI_WAIT_D2H:
2438                 return atapi_data_tc_completion_handler(ireq, completion_code);
2439
2440         default:
2441                 dev_warn(&ihost->pdev->dev, "%s: %x in wrong state %s\n",
2442                          __func__, completion_code, req_state_name(state));
2443                 return SCI_FAILURE_INVALID_STATE;
2444         }
2445 }
2446
2447 /**
2448  * isci_request_process_response_iu() - This function sets the status and
2449  *    response iu, in the task struct, from the request object for the upper
2450  *    layer driver.
2451  * @task: This parameter is the task struct from the upper layer driver.
2452  * @resp_iu: This parameter points to the response iu of the completed request.
2453  * @dev: This parameter specifies the linux device struct.
2454  *
2455  * none.
2456  */
2457 static void isci_request_process_response_iu(
2458         struct sas_task *task,
2459         struct ssp_response_iu *resp_iu,
2460         struct device *dev)
2461 {
2462         dev_dbg(dev,
2463                 "%s: resp_iu = %p "
2464                 "resp_iu->status = 0x%x,\nresp_iu->datapres = %d "
2465                 "resp_iu->response_data_len = %x, "
2466                 "resp_iu->sense_data_len = %x\nresponse data: ",
2467                 __func__,
2468                 resp_iu,
2469                 resp_iu->status,
2470                 resp_iu->datapres,
2471                 resp_iu->response_data_len,
2472                 resp_iu->sense_data_len);
2473
2474         task->task_status.stat = resp_iu->status;
2475
2476         /* libsas updates the task status fields based on the response iu. */
2477         sas_ssp_task_response(dev, task, resp_iu);
2478 }
2479
2480 /**
2481  * isci_request_set_open_reject_status() - This function prepares the I/O
2482  *    completion for OPEN_REJECT conditions.
2483  * @request: This parameter is the completed isci_request object.
2484  * @task: This parameter is the task struct from the upper layer driver.
2485  * @response_ptr: This parameter specifies the service response for the I/O.
2486  * @status_ptr: This parameter specifies the exec status for the I/O.
2487  * @open_rej_reason: This parameter specifies the encoded reason for the
2488  *    abandon-class reject.
2489  *
2490  * none.
2491  */
2492 static void isci_request_set_open_reject_status(
2493         struct isci_request *request,
2494         struct sas_task *task,
2495         enum service_response *response_ptr,
2496         enum exec_status *status_ptr,
2497         enum sas_open_rej_reason open_rej_reason)
2498 {
2499         /* Task in the target is done. */
2500         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2501         *response_ptr                     = SAS_TASK_UNDELIVERED;
2502         *status_ptr                       = SAS_OPEN_REJECT;
2503         task->task_status.open_rej_reason = open_rej_reason;
2504 }
2505
2506 /**
2507  * isci_request_handle_controller_specific_errors() - This function decodes
2508  *    controller-specific I/O completion error conditions.
2509  * @idev: Remote device
2510  * @request: This parameter is the completed isci_request object.
2511  * @task: This parameter is the task struct from the upper layer driver.
2512  * @response_ptr: This parameter specifies the service response for the I/O.
2513  * @status_ptr: This parameter specifies the exec status for the I/O.
2514  *
2515  * none.
2516  */
2517 static void isci_request_handle_controller_specific_errors(
2518         struct isci_remote_device *idev,
2519         struct isci_request *request,
2520         struct sas_task *task,
2521         enum service_response *response_ptr,
2522         enum exec_status *status_ptr)
2523 {
2524         unsigned int cstatus;
2525
2526         cstatus = request->scu_status;
2527
2528         dev_dbg(&request->isci_host->pdev->dev,
2529                 "%s: %p SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR "
2530                 "- controller status = 0x%x\n",
2531                 __func__, request, cstatus);
2532
2533         /* Decode the controller-specific errors; most
2534          * important is to recognize those conditions in which
2535          * the target may still have a task outstanding that
2536          * must be aborted.
2537          *
2538          * Note that there are SCU completion codes being
2539          * named in the decode below for which SCIC has already
2540          * done work to handle them in a way other than as
2541          * a controller-specific completion code; these are left
2542          * in the decode below for completeness sake.
2543          */
2544         switch (cstatus) {
2545         case SCU_TASK_DONE_DMASETUP_DIRERR:
2546         /* Also SCU_TASK_DONE_SMP_FRM_TYPE_ERR: */
2547         case SCU_TASK_DONE_XFERCNT_ERR:
2548                 /* Also SCU_TASK_DONE_SMP_UFI_ERR: */
2549                 if (task->task_proto == SAS_PROTOCOL_SMP) {
2550                         /* SCU_TASK_DONE_SMP_UFI_ERR == Task Done. */
2551                         *response_ptr = SAS_TASK_COMPLETE;
2552
2553                         /* See if the device has been/is being stopped. Note
2554                          * that we ignore the quiesce state, since we are
2555                          * concerned about the actual device state.
2556                          */
2557                         if (!idev)
2558                                 *status_ptr = SAS_DEVICE_UNKNOWN;
2559                         else
2560                                 *status_ptr = SAS_ABORTED_TASK;
2561
2562                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2563                 } else {
2564                         /* Task in the target is not done. */
2565                         *response_ptr = SAS_TASK_UNDELIVERED;
2566
2567                         if (!idev)
2568                                 *status_ptr = SAS_DEVICE_UNKNOWN;
2569                         else
2570                                 *status_ptr = SAS_SAM_STAT_TASK_ABORTED;
2571
2572                         clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2573                 }
2574
2575                 break;
2576
2577         case SCU_TASK_DONE_CRC_ERR:
2578         case SCU_TASK_DONE_NAK_CMD_ERR:
2579         case SCU_TASK_DONE_EXCESS_DATA:
2580         case SCU_TASK_DONE_UNEXP_FIS:
2581         /* Also SCU_TASK_DONE_UNEXP_RESP: */
2582         case SCU_TASK_DONE_VIIT_ENTRY_NV:       /* TODO - conditions? */
2583         case SCU_TASK_DONE_IIT_ENTRY_NV:        /* TODO - conditions? */
2584         case SCU_TASK_DONE_RNCNV_OUTBOUND:      /* TODO - conditions? */
2585                 /* These are conditions in which the target
2586                  * has completed the task, so that no cleanup
2587                  * is necessary.
2588                  */
2589                 *response_ptr = SAS_TASK_COMPLETE;
2590
2591                 /* See if the device has been/is being stopped. Note
2592                  * that we ignore the quiesce state, since we are
2593                  * concerned about the actual device state.
2594                  */
2595                 if (!idev)
2596                         *status_ptr = SAS_DEVICE_UNKNOWN;
2597                 else
2598                         *status_ptr = SAS_ABORTED_TASK;
2599
2600                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2601                 break;
2602
2603
2604         /* Note that the only open reject completion codes seen here will be
2605          * abandon-class codes; all others are automatically retried in the SCU.
2606          */
2607         case SCU_TASK_OPEN_REJECT_WRONG_DESTINATION:
2608
2609                 isci_request_set_open_reject_status(
2610                         request, task, response_ptr, status_ptr,
2611                         SAS_OREJ_WRONG_DEST);
2612                 break;
2613
2614         case SCU_TASK_OPEN_REJECT_ZONE_VIOLATION:
2615
2616                 /* Note - the return of AB0 will change when
2617                  * libsas implements detection of zone violations.
2618                  */
2619                 isci_request_set_open_reject_status(
2620                         request, task, response_ptr, status_ptr,
2621                         SAS_OREJ_RESV_AB0);
2622                 break;
2623
2624         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_1:
2625
2626                 isci_request_set_open_reject_status(
2627                         request, task, response_ptr, status_ptr,
2628                         SAS_OREJ_RESV_AB1);
2629                 break;
2630
2631         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_2:
2632
2633                 isci_request_set_open_reject_status(
2634                         request, task, response_ptr, status_ptr,
2635                         SAS_OREJ_RESV_AB2);
2636                 break;
2637
2638         case SCU_TASK_OPEN_REJECT_RESERVED_ABANDON_3:
2639
2640                 isci_request_set_open_reject_status(
2641                         request, task, response_ptr, status_ptr,
2642                         SAS_OREJ_RESV_AB3);
2643                 break;
2644
2645         case SCU_TASK_OPEN_REJECT_BAD_DESTINATION:
2646
2647                 isci_request_set_open_reject_status(
2648                         request, task, response_ptr, status_ptr,
2649                         SAS_OREJ_BAD_DEST);
2650                 break;
2651
2652         case SCU_TASK_OPEN_REJECT_STP_RESOURCES_BUSY:
2653
2654                 isci_request_set_open_reject_status(
2655                         request, task, response_ptr, status_ptr,
2656                         SAS_OREJ_STP_NORES);
2657                 break;
2658
2659         case SCU_TASK_OPEN_REJECT_PROTOCOL_NOT_SUPPORTED:
2660
2661                 isci_request_set_open_reject_status(
2662                         request, task, response_ptr, status_ptr,
2663                         SAS_OREJ_EPROTO);
2664                 break;
2665
2666         case SCU_TASK_OPEN_REJECT_CONNECTION_RATE_NOT_SUPPORTED:
2667
2668                 isci_request_set_open_reject_status(
2669                         request, task, response_ptr, status_ptr,
2670                         SAS_OREJ_CONN_RATE);
2671                 break;
2672
2673         case SCU_TASK_DONE_LL_R_ERR:
2674         /* Also SCU_TASK_DONE_ACK_NAK_TO: */
2675         case SCU_TASK_DONE_LL_PERR:
2676         case SCU_TASK_DONE_LL_SY_TERM:
2677         /* Also SCU_TASK_DONE_NAK_ERR:*/
2678         case SCU_TASK_DONE_LL_LF_TERM:
2679         /* Also SCU_TASK_DONE_DATA_LEN_ERR: */
2680         case SCU_TASK_DONE_LL_ABORT_ERR:
2681         case SCU_TASK_DONE_SEQ_INV_TYPE:
2682         /* Also SCU_TASK_DONE_UNEXP_XR: */
2683         case SCU_TASK_DONE_XR_IU_LEN_ERR:
2684         case SCU_TASK_DONE_INV_FIS_LEN:
2685         /* Also SCU_TASK_DONE_XR_WD_LEN: */
2686         case SCU_TASK_DONE_SDMA_ERR:
2687         case SCU_TASK_DONE_OFFSET_ERR:
2688         case SCU_TASK_DONE_MAX_PLD_ERR:
2689         case SCU_TASK_DONE_LF_ERR:
2690         case SCU_TASK_DONE_SMP_RESP_TO_ERR:  /* Escalate to dev reset? */
2691         case SCU_TASK_DONE_SMP_LL_RX_ERR:
2692         case SCU_TASK_DONE_UNEXP_DATA:
2693         case SCU_TASK_DONE_UNEXP_SDBFIS:
2694         case SCU_TASK_DONE_REG_ERR:
2695         case SCU_TASK_DONE_SDB_ERR:
2696         case SCU_TASK_DONE_TASK_ABORT:
2697         default:
2698                 /* Task in the target is not done. */
2699                 *response_ptr = SAS_TASK_UNDELIVERED;
2700                 *status_ptr = SAS_SAM_STAT_TASK_ABORTED;
2701
2702                 if (task->task_proto == SAS_PROTOCOL_SMP)
2703                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2704                 else
2705                         clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2706                 break;
2707         }
2708 }
2709
2710 static void isci_process_stp_response(struct sas_task *task, struct dev_to_host_fis *fis)
2711 {
2712         struct task_status_struct *ts = &task->task_status;
2713         struct ata_task_resp *resp = (void *)&ts->buf[0];
2714
2715         resp->frame_len = sizeof(*fis);
2716         memcpy(resp->ending_fis, fis, sizeof(*fis));
2717         ts->buf_valid_size = sizeof(*resp);
2718
2719         /* If an error is flagged let libata decode the fis */
2720         if (ac_err_mask(fis->status))
2721                 ts->stat = SAS_PROTO_RESPONSE;
2722         else
2723                 ts->stat = SAS_SAM_STAT_GOOD;
2724
2725         ts->resp = SAS_TASK_COMPLETE;
2726 }
2727
2728 static void isci_request_io_request_complete(struct isci_host *ihost,
2729                                              struct isci_request *request,
2730                                              enum sci_io_status completion_status)
2731 {
2732         struct sas_task *task = isci_request_access_task(request);
2733         struct ssp_response_iu *resp_iu;
2734         unsigned long task_flags;
2735         struct isci_remote_device *idev = request->target_device;
2736         enum service_response response = SAS_TASK_UNDELIVERED;
2737         enum exec_status status = SAS_ABORTED_TASK;
2738
2739         dev_dbg(&ihost->pdev->dev,
2740                 "%s: request = %p, task = %p, "
2741                 "task->data_dir = %d completion_status = 0x%x\n",
2742                 __func__, request, task, task->data_dir, completion_status);
2743
2744         /* The request is done from an SCU HW perspective. */
2745
2746         /* This is an active request being completed from the core. */
2747         switch (completion_status) {
2748
2749         case SCI_IO_FAILURE_RESPONSE_VALID:
2750                 dev_dbg(&ihost->pdev->dev,
2751                         "%s: SCI_IO_FAILURE_RESPONSE_VALID (%p/%p)\n",
2752                         __func__, request, task);
2753
2754                 if (sas_protocol_ata(task->task_proto)) {
2755                         isci_process_stp_response(task, &request->stp.rsp);
2756                 } else if (SAS_PROTOCOL_SSP == task->task_proto) {
2757
2758                         /* crack the iu response buffer. */
2759                         resp_iu = &request->ssp.rsp;
2760                         isci_request_process_response_iu(task, resp_iu,
2761                                                          &ihost->pdev->dev);
2762
2763                 } else if (SAS_PROTOCOL_SMP == task->task_proto) {
2764
2765                         dev_err(&ihost->pdev->dev,
2766                                 "%s: SCI_IO_FAILURE_RESPONSE_VALID: "
2767                                         "SAS_PROTOCOL_SMP protocol\n",
2768                                 __func__);
2769
2770                 } else
2771                         dev_err(&ihost->pdev->dev,
2772                                 "%s: unknown protocol\n", __func__);
2773
2774                 /* use the task status set in the task struct by the
2775                 * isci_request_process_response_iu call.
2776                 */
2777                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2778                 response = task->task_status.resp;
2779                 status = task->task_status.stat;
2780                 break;
2781
2782         case SCI_IO_SUCCESS:
2783         case SCI_IO_SUCCESS_IO_DONE_EARLY:
2784
2785                 response = SAS_TASK_COMPLETE;
2786                 status   = SAS_SAM_STAT_GOOD;
2787                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2788
2789                 if (completion_status == SCI_IO_SUCCESS_IO_DONE_EARLY) {
2790
2791                         /* This was an SSP / STP / SATA transfer.
2792                         * There is a possibility that less data than
2793                         * the maximum was transferred.
2794                         */
2795                         u32 transferred_length = sci_req_tx_bytes(request);
2796
2797                         task->task_status.residual
2798                                 = task->total_xfer_len - transferred_length;
2799
2800                         /* If there were residual bytes, call this an
2801                         * underrun.
2802                         */
2803                         if (task->task_status.residual != 0)
2804                                 status = SAS_DATA_UNDERRUN;
2805
2806                         dev_dbg(&ihost->pdev->dev,
2807                                 "%s: SCI_IO_SUCCESS_IO_DONE_EARLY %d\n",
2808                                 __func__, status);
2809
2810                 } else
2811                         dev_dbg(&ihost->pdev->dev, "%s: SCI_IO_SUCCESS\n",
2812                                 __func__);
2813                 break;
2814
2815         case SCI_IO_FAILURE_TERMINATED:
2816
2817                 dev_dbg(&ihost->pdev->dev,
2818                         "%s: SCI_IO_FAILURE_TERMINATED (%p/%p)\n",
2819                         __func__, request, task);
2820
2821                 /* The request was terminated explicitly. */
2822                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2823                 response = SAS_TASK_UNDELIVERED;
2824
2825                 /* See if the device has been/is being stopped. Note
2826                 * that we ignore the quiesce state, since we are
2827                 * concerned about the actual device state.
2828                 */
2829                 if (!idev)
2830                         status = SAS_DEVICE_UNKNOWN;
2831                 else
2832                         status = SAS_ABORTED_TASK;
2833                 break;
2834
2835         case SCI_FAILURE_CONTROLLER_SPECIFIC_IO_ERR:
2836
2837                 isci_request_handle_controller_specific_errors(idev, request,
2838                                                                task, &response,
2839                                                                &status);
2840                 break;
2841
2842         case SCI_IO_FAILURE_REMOTE_DEVICE_RESET_REQUIRED:
2843                 /* This is a special case, in that the I/O completion
2844                 * is telling us that the device needs a reset.
2845                 * In order for the device reset condition to be
2846                 * noticed, the I/O has to be handled in the error
2847                 * handler.  Set the reset flag and cause the
2848                 * SCSI error thread to be scheduled.
2849                 */
2850                 spin_lock_irqsave(&task->task_state_lock, task_flags);
2851                 task->task_state_flags |= SAS_TASK_NEED_DEV_RESET;
2852                 spin_unlock_irqrestore(&task->task_state_lock, task_flags);
2853
2854                 /* Fail the I/O. */
2855                 response = SAS_TASK_UNDELIVERED;
2856                 status = SAS_SAM_STAT_TASK_ABORTED;
2857
2858                 clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2859                 break;
2860
2861         case SCI_FAILURE_RETRY_REQUIRED:
2862
2863                 /* Fail the I/O so it can be retried. */
2864                 response = SAS_TASK_UNDELIVERED;
2865                 if (!idev)
2866                         status = SAS_DEVICE_UNKNOWN;
2867                 else
2868                         status = SAS_ABORTED_TASK;
2869
2870                 set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2871                 break;
2872
2873
2874         default:
2875                 /* Catch any otherwise unhandled error codes here. */
2876                 dev_dbg(&ihost->pdev->dev,
2877                         "%s: invalid completion code: 0x%x - "
2878                                 "isci_request = %p\n",
2879                         __func__, completion_status, request);
2880
2881                 response = SAS_TASK_UNDELIVERED;
2882
2883                 /* See if the device has been/is being stopped. Note
2884                 * that we ignore the quiesce state, since we are
2885                 * concerned about the actual device state.
2886                 */
2887                 if (!idev)
2888                         status = SAS_DEVICE_UNKNOWN;
2889                 else
2890                         status = SAS_ABORTED_TASK;
2891
2892                 if (SAS_PROTOCOL_SMP == task->task_proto)
2893                         set_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2894                 else
2895                         clear_bit(IREQ_COMPLETE_IN_TARGET, &request->flags);
2896                 break;
2897         }
2898
2899         switch (task->task_proto) {
2900         case SAS_PROTOCOL_SSP:
2901                 if (task->data_dir == DMA_NONE)
2902                         break;
2903                 if (task->num_scatter == 0)
2904                         /* 0 indicates a single dma address */
2905                         dma_unmap_single(&ihost->pdev->dev,
2906                                          request->zero_scatter_daddr,
2907                                          task->total_xfer_len, task->data_dir);
2908                 else  /* unmap the sgl dma addresses */
2909                         dma_unmap_sg(&ihost->pdev->dev, task->scatter,
2910                                      request->num_sg_entries, task->data_dir);
2911                 break;
2912         case SAS_PROTOCOL_SMP: {
2913                 struct scatterlist *sg = &task->smp_task.smp_req;
2914                 struct smp_req *smp_req;
2915                 void *kaddr;
2916
2917                 dma_unmap_sg(&ihost->pdev->dev, sg, 1, DMA_TO_DEVICE);
2918
2919                 /* need to swab it back in case the command buffer is re-used */
2920                 kaddr = kmap_atomic(sg_page(sg));
2921                 smp_req = kaddr + sg->offset;
2922                 sci_swab32_cpy(smp_req, smp_req, sg->length / sizeof(u32));
2923                 kunmap_atomic(kaddr);
2924                 break;
2925         }
2926         default:
2927                 break;
2928         }
2929
2930         spin_lock_irqsave(&task->task_state_lock, task_flags);
2931
2932         task->task_status.resp = response;
2933         task->task_status.stat = status;
2934
2935         if (test_bit(IREQ_COMPLETE_IN_TARGET, &request->flags)) {
2936                 /* Normal notification (task_done) */
2937                 task->task_state_flags |= SAS_TASK_STATE_DONE;
2938                 task->task_state_flags &= ~SAS_TASK_STATE_PENDING;
2939         }
2940         spin_unlock_irqrestore(&task->task_state_lock, task_flags);
2941
2942         /* complete the io request to the core. */
2943         sci_controller_complete_io(ihost, request->target_device, request);
2944
2945         /* set terminated handle so it cannot be completed or
2946          * terminated again, and to cause any calls into abort
2947          * task to recognize the already completed case.
2948          */
2949         set_bit(IREQ_TERMINATED, &request->flags);
2950
2951         ireq_done(ihost, request, task);
2952 }
2953
2954 static void sci_request_started_state_enter(struct sci_base_state_machine *sm)
2955 {
2956         struct isci_request *ireq = container_of(sm, typeof(*ireq), sm);
2957         struct domain_device *dev = ireq->target_device->domain_dev;
2958         enum sci_base_request_states state;
2959         struct sas_task *task;
2960
2961         /* XXX as hch said always creating an internal sas_task for tmf
2962          * requests would simplify the driver
2963          */
2964         task = (test_bit(IREQ_TMF, &ireq->flags)) ? NULL : isci_request_access_task(ireq);
2965
2966         /* all unaccelerated request types (non ssp or ncq) handled with
2967          * substates
2968          */
2969         if (!task && dev->dev_type == SAS_END_DEVICE) {
2970                 state = SCI_REQ_TASK_WAIT_TC_COMP;
2971         } else if (task && task->task_proto == SAS_PROTOCOL_SMP) {
2972                 state = SCI_REQ_SMP_WAIT_RESP;
2973         } else if (task && sas_protocol_ata(task->task_proto) &&
2974                    !task->ata_task.use_ncq) {
2975                 if (dev->sata_dev.class == ATA_DEV_ATAPI &&
2976                         task->ata_task.fis.command == ATA_CMD_PACKET) {
2977                         state = SCI_REQ_ATAPI_WAIT_H2D;
2978                 } else if (task->data_dir == DMA_NONE) {
2979                         state = SCI_REQ_STP_NON_DATA_WAIT_H2D;
2980                 } else if (task->ata_task.dma_xfer) {
2981                         state = SCI_REQ_STP_UDMA_WAIT_TC_COMP;
2982                 } else /* PIO */ {
2983                         state = SCI_REQ_STP_PIO_WAIT_H2D;
2984                 }
2985         } else {
2986                 /* SSP or NCQ are fully accelerated, no substates */
2987                 return;
2988         }
2989         sci_change_state(sm, state);
2990 }
2991
2992 static void sci_request_completed_state_enter(struct sci_base_state_machine *sm)
2993 {
2994         struct isci_request *ireq = container_of(sm, typeof(*ireq), sm);
2995         struct isci_host *ihost = ireq->owning_controller;
2996
2997         /* Tell the SCI_USER that the IO request is complete */
2998         if (!test_bit(IREQ_TMF, &ireq->flags))
2999                 isci_request_io_request_complete(ihost, ireq,
3000                                                  ireq->sci_status);
3001         else
3002                 isci_task_request_complete(ihost, ireq, ireq->sci_status);
3003 }
3004
3005 static void sci_request_aborting_state_enter(struct sci_base_state_machine *sm)
3006 {
3007         struct isci_request *ireq = container_of(sm, typeof(*ireq), sm);
3008
3009         /* Setting the abort bit in the Task Context is required by the silicon. */
3010         ireq->tc->abort = 1;
3011 }
3012
3013 static void sci_stp_request_started_non_data_await_h2d_completion_enter(struct sci_base_state_machine *sm)
3014 {
3015         struct isci_request *ireq = container_of(sm, typeof(*ireq), sm);
3016
3017         ireq->target_device->working_request = ireq;
3018 }
3019
3020 static void sci_stp_request_started_pio_await_h2d_completion_enter(struct sci_base_state_machine *sm)
3021 {
3022         struct isci_request *ireq = container_of(sm, typeof(*ireq), sm);
3023
3024         ireq->target_device->working_request = ireq;
3025 }
3026
3027 static const struct sci_base_state sci_request_state_table[] = {
3028         [SCI_REQ_INIT] = { },
3029         [SCI_REQ_CONSTRUCTED] = { },
3030         [SCI_REQ_STARTED] = {
3031                 .enter_state = sci_request_started_state_enter,
3032         },
3033         [SCI_REQ_STP_NON_DATA_WAIT_H2D] = {
3034                 .enter_state = sci_stp_request_started_non_data_await_h2d_completion_enter,
3035         },
3036         [SCI_REQ_STP_NON_DATA_WAIT_D2H] = { },
3037         [SCI_REQ_STP_PIO_WAIT_H2D] = {
3038                 .enter_state = sci_stp_request_started_pio_await_h2d_completion_enter,
3039         },
3040         [SCI_REQ_STP_PIO_WAIT_FRAME] = { },
3041         [SCI_REQ_STP_PIO_DATA_IN] = { },
3042         [SCI_REQ_STP_PIO_DATA_OUT] = { },
3043         [SCI_REQ_STP_UDMA_WAIT_TC_COMP] = { },
3044         [SCI_REQ_STP_UDMA_WAIT_D2H] = { },
3045         [SCI_REQ_TASK_WAIT_TC_COMP] = { },
3046         [SCI_REQ_TASK_WAIT_TC_RESP] = { },
3047         [SCI_REQ_SMP_WAIT_RESP] = { },
3048         [SCI_REQ_SMP_WAIT_TC_COMP] = { },
3049         [SCI_REQ_ATAPI_WAIT_H2D] = { },
3050         [SCI_REQ_ATAPI_WAIT_PIO_SETUP] = { },
3051         [SCI_REQ_ATAPI_WAIT_D2H] = { },
3052         [SCI_REQ_ATAPI_WAIT_TC_COMP] = { },
3053         [SCI_REQ_COMPLETED] = {
3054                 .enter_state = sci_request_completed_state_enter,
3055         },
3056         [SCI_REQ_ABORTING] = {
3057                 .enter_state = sci_request_aborting_state_enter,
3058         },
3059         [SCI_REQ_FINAL] = { },
3060 };
3061
3062 static void
3063 sci_general_request_construct(struct isci_host *ihost,
3064                                    struct isci_remote_device *idev,
3065                                    struct isci_request *ireq)
3066 {
3067         sci_init_sm(&ireq->sm, sci_request_state_table, SCI_REQ_INIT);
3068
3069         ireq->target_device = idev;
3070         ireq->protocol = SAS_PROTOCOL_NONE;
3071         ireq->saved_rx_frame_index = SCU_INVALID_FRAME_INDEX;
3072
3073         ireq->sci_status   = SCI_SUCCESS;
3074         ireq->scu_status   = 0;
3075         ireq->post_context = 0xFFFFFFFF;
3076 }
3077
3078 static enum sci_status
3079 sci_io_request_construct(struct isci_host *ihost,
3080                           struct isci_remote_device *idev,
3081                           struct isci_request *ireq)
3082 {
3083         struct domain_device *dev = idev->domain_dev;
3084         enum sci_status status = SCI_SUCCESS;
3085
3086         /* Build the common part of the request */
3087         sci_general_request_construct(ihost, idev, ireq);
3088
3089         if (idev->rnc.remote_node_index == SCIC_SDS_REMOTE_NODE_CONTEXT_INVALID_INDEX)
3090                 return SCI_FAILURE_INVALID_REMOTE_DEVICE;
3091
3092         if (dev->dev_type == SAS_END_DEVICE)
3093                 /* pass */;
3094         else if (dev_is_sata(dev))
3095                 memset(&ireq->stp.cmd, 0, sizeof(ireq->stp.cmd));
3096         else if (dev_is_expander(dev->dev_type))
3097                 /* pass */;
3098         else
3099                 return SCI_FAILURE_UNSUPPORTED_PROTOCOL;
3100
3101         memset(ireq->tc, 0, offsetof(struct scu_task_context, sgl_pair_ab));
3102
3103         return status;
3104 }
3105
3106 enum sci_status sci_task_request_construct(struct isci_host *ihost,
3107                                             struct isci_remote_device *idev,
3108                                             u16 io_tag, struct isci_request *ireq)
3109 {
3110         struct domain_device *dev = idev->domain_dev;
3111         enum sci_status status = SCI_SUCCESS;
3112
3113         /* Build the common part of the request */
3114         sci_general_request_construct(ihost, idev, ireq);
3115
3116         if (dev->dev_type == SAS_END_DEVICE || dev_is_sata(dev)) {
3117                 set_bit(IREQ_TMF, &ireq->flags);
3118                 memset(ireq->tc, 0, sizeof(struct scu_task_context));
3119
3120                 /* Set the protocol indicator. */
3121                 if (dev_is_sata(dev))
3122                         ireq->protocol = SAS_PROTOCOL_STP;
3123                 else
3124                         ireq->protocol = SAS_PROTOCOL_SSP;
3125         } else
3126                 status = SCI_FAILURE_UNSUPPORTED_PROTOCOL;
3127
3128         return status;
3129 }
3130
3131 static enum sci_status isci_request_ssp_request_construct(
3132         struct isci_request *request)
3133 {
3134         enum sci_status status;
3135
3136         dev_dbg(&request->isci_host->pdev->dev,
3137                 "%s: request = %p\n",
3138                 __func__,
3139                 request);
3140         status = sci_io_request_construct_basic_ssp(request);
3141         return status;
3142 }
3143
3144 static enum sci_status isci_request_stp_request_construct(struct isci_request *ireq)
3145 {
3146         struct sas_task *task = isci_request_access_task(ireq);
3147         struct host_to_dev_fis *fis = &ireq->stp.cmd;
3148         struct ata_queued_cmd *qc = task->uldd_task;
3149         enum sci_status status;
3150
3151         dev_dbg(&ireq->isci_host->pdev->dev,
3152                 "%s: ireq = %p\n",
3153                 __func__,
3154                 ireq);
3155
3156         memcpy(fis, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
3157         if (!task->ata_task.device_control_reg_update)
3158                 fis->flags |= 0x80;
3159         fis->flags &= 0xF0;
3160
3161         status = sci_io_request_construct_basic_sata(ireq);
3162
3163         if (qc && (qc->tf.command == ATA_CMD_FPDMA_WRITE ||
3164                    qc->tf.command == ATA_CMD_FPDMA_READ ||
3165                    qc->tf.command == ATA_CMD_FPDMA_RECV ||
3166                    qc->tf.command == ATA_CMD_FPDMA_SEND ||
3167                    qc->tf.command == ATA_CMD_NCQ_NON_DATA)) {
3168                 fis->sector_count = qc->tag << 3;
3169                 ireq->tc->type.stp.ncq_tag = qc->tag;
3170         }
3171
3172         return status;
3173 }
3174
3175 static enum sci_status
3176 sci_io_request_construct_smp(struct device *dev,
3177                               struct isci_request *ireq,
3178                               struct sas_task *task)
3179 {
3180         struct scatterlist *sg = &task->smp_task.smp_req;
3181         struct isci_remote_device *idev;
3182         struct scu_task_context *task_context;
3183         struct isci_port *iport;
3184         struct smp_req *smp_req;
3185         void *kaddr;
3186         u8 req_len;
3187         u32 cmd;
3188
3189         kaddr = kmap_atomic(sg_page(sg));
3190         smp_req = kaddr + sg->offset;
3191         /*
3192          * Look at the SMP requests' header fields; for certain SAS 1.x SMP
3193          * functions under SAS 2.0, a zero request length really indicates
3194          * a non-zero default length.
3195          */
3196         if (smp_req->req_len == 0) {
3197                 switch (smp_req->func) {
3198                 case SMP_DISCOVER:
3199                 case SMP_REPORT_PHY_ERR_LOG:
3200                 case SMP_REPORT_PHY_SATA:
3201                 case SMP_REPORT_ROUTE_INFO:
3202                         smp_req->req_len = 2;
3203                         break;
3204                 case SMP_CONF_ROUTE_INFO:
3205                 case SMP_PHY_CONTROL:
3206                 case SMP_PHY_TEST_FUNCTION:
3207                         smp_req->req_len = 9;
3208                         break;
3209                         /* Default - zero is a valid default for 2.0. */
3210                 }
3211         }
3212         req_len = smp_req->req_len;
3213         sci_swab32_cpy(smp_req, smp_req, sg->length / sizeof(u32));
3214         cmd = *(u32 *) smp_req;
3215         kunmap_atomic(kaddr);
3216
3217         if (!dma_map_sg(dev, sg, 1, DMA_TO_DEVICE))
3218                 return SCI_FAILURE;
3219
3220         ireq->protocol = SAS_PROTOCOL_SMP;
3221
3222         /* byte swap the smp request. */
3223
3224         task_context = ireq->tc;
3225
3226         idev = ireq->target_device;
3227         iport = idev->owning_port;
3228
3229         /*
3230          * Fill in the TC with its required data
3231          * 00h
3232          */
3233         task_context->priority = 0;
3234         task_context->initiator_request = 1;
3235         task_context->connection_rate = idev->connection_rate;
3236         task_context->protocol_engine_index = ISCI_PEG;
3237         task_context->logical_port_index = iport->physical_port_index;
3238         task_context->protocol_type = SCU_TASK_CONTEXT_PROTOCOL_SMP;
3239         task_context->abort = 0;
3240         task_context->valid = SCU_TASK_CONTEXT_VALID;
3241         task_context->context_type = SCU_TASK_CONTEXT_TYPE;
3242
3243         /* 04h */
3244         task_context->remote_node_index = idev->rnc.remote_node_index;
3245         task_context->command_code = 0;
3246         task_context->task_type = SCU_TASK_TYPE_SMP_REQUEST;
3247
3248         /* 08h */
3249         task_context->link_layer_control = 0;
3250         task_context->do_not_dma_ssp_good_response = 1;
3251         task_context->strict_ordering = 0;
3252         task_context->control_frame = 1;
3253         task_context->timeout_enable = 0;
3254         task_context->block_guard_enable = 0;
3255
3256         /* 0ch */
3257         task_context->address_modifier = 0;
3258
3259         /* 10h */
3260         task_context->ssp_command_iu_length = req_len;
3261
3262         /* 14h */
3263         task_context->transfer_length_bytes = 0;
3264
3265         /*
3266          * 18h ~ 30h, protocol specific
3267          * since commandIU has been build by framework at this point, we just
3268          * copy the frist DWord from command IU to this location. */
3269         memcpy(&task_context->type.smp, &cmd, sizeof(u32));
3270
3271         /*
3272          * 40h
3273          * "For SMP you could program it to zero. We would prefer that way
3274          * so that done code will be consistent." - Venki
3275          */
3276         task_context->task_phase = 0;
3277
3278         ireq->post_context = (SCU_CONTEXT_COMMAND_REQUEST_TYPE_POST_TC |
3279                               (ISCI_PEG << SCU_CONTEXT_COMMAND_PROTOCOL_ENGINE_GROUP_SHIFT) |
3280                                (iport->physical_port_index <<
3281                                 SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT) |
3282                               ISCI_TAG_TCI(ireq->io_tag));
3283         /*
3284          * Copy the physical address for the command buffer to the SCU Task
3285          * Context command buffer should not contain command header.
3286          */
3287         task_context->command_iu_upper = upper_32_bits(sg_dma_address(sg));
3288         task_context->command_iu_lower = lower_32_bits(sg_dma_address(sg) + sizeof(u32));
3289
3290         /* SMP response comes as UF, so no need to set response IU address. */
3291         task_context->response_iu_upper = 0;
3292         task_context->response_iu_lower = 0;
3293
3294         sci_change_state(&ireq->sm, SCI_REQ_CONSTRUCTED);
3295
3296         return SCI_SUCCESS;
3297 }
3298
3299 /*
3300  * isci_smp_request_build() - This function builds the smp request.
3301  * @ireq: This parameter points to the isci_request allocated in the
3302  *    request construct function.
3303  *
3304  * SCI_SUCCESS on successfull completion, or specific failure code.
3305  */
3306 static enum sci_status isci_smp_request_build(struct isci_request *ireq)
3307 {
3308         struct sas_task *task = isci_request_access_task(ireq);
3309         struct device *dev = &ireq->isci_host->pdev->dev;
3310         enum sci_status status = SCI_FAILURE;
3311
3312         status = sci_io_request_construct_smp(dev, ireq, task);
3313         if (status != SCI_SUCCESS)
3314                 dev_dbg(&ireq->isci_host->pdev->dev,
3315                          "%s: failed with status = %d\n",
3316                          __func__,
3317                          status);
3318
3319         return status;
3320 }
3321
3322 /**
3323  * isci_io_request_build() - This function builds the io request object.
3324  * @ihost: This parameter specifies the ISCI host object
3325  * @request: This parameter points to the isci_request object allocated in the
3326  *    request construct function.
3327  * @idev: This parameter is the handle for the sci core's remote device
3328  *    object that is the destination for this request.
3329  *
3330  * SCI_SUCCESS on successfull completion, or specific failure code.
3331  */
3332 static enum sci_status isci_io_request_build(struct isci_host *ihost,
3333                                              struct isci_request *request,
3334                                              struct isci_remote_device *idev)
3335 {
3336         enum sci_status status = SCI_SUCCESS;
3337         struct sas_task *task = isci_request_access_task(request);
3338
3339         dev_dbg(&ihost->pdev->dev,
3340                 "%s: idev = 0x%p; request = %p, "
3341                 "num_scatter = %d\n",
3342                 __func__,
3343                 idev,
3344                 request,
3345                 task->num_scatter);
3346
3347         /* map the sgl addresses, if present.
3348          * libata does the mapping for sata devices
3349          * before we get the request.
3350          */
3351         if (task->num_scatter &&
3352             !sas_protocol_ata(task->task_proto) &&
3353             !(SAS_PROTOCOL_SMP & task->task_proto)) {
3354
3355                 request->num_sg_entries = dma_map_sg(
3356                         &ihost->pdev->dev,
3357                         task->scatter,
3358                         task->num_scatter,
3359                         task->data_dir
3360                         );
3361
3362                 if (request->num_sg_entries == 0)
3363                         return SCI_FAILURE_INSUFFICIENT_RESOURCES;
3364         }
3365
3366         status = sci_io_request_construct(ihost, idev, request);
3367
3368         if (status != SCI_SUCCESS) {
3369                 dev_dbg(&ihost->pdev->dev,
3370                          "%s: failed request construct\n",
3371                          __func__);
3372                 return SCI_FAILURE;
3373         }
3374
3375         switch (task->task_proto) {
3376         case SAS_PROTOCOL_SMP:
3377                 status = isci_smp_request_build(request);
3378                 break;
3379         case SAS_PROTOCOL_SSP:
3380                 status = isci_request_ssp_request_construct(request);
3381                 break;
3382         case SAS_PROTOCOL_SATA:
3383         case SAS_PROTOCOL_STP:
3384         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
3385                 status = isci_request_stp_request_construct(request);
3386                 break;
3387         default:
3388                 dev_dbg(&ihost->pdev->dev,
3389                          "%s: unknown protocol\n", __func__);
3390                 return SCI_FAILURE;
3391         }
3392
3393         return SCI_SUCCESS;
3394 }
3395
3396 static struct isci_request *isci_request_from_tag(struct isci_host *ihost, u16 tag)
3397 {
3398         struct isci_request *ireq;
3399
3400         ireq = ihost->reqs[ISCI_TAG_TCI(tag)];
3401         ireq->io_tag = tag;
3402         ireq->io_request_completion = NULL;
3403         ireq->flags = 0;
3404         ireq->num_sg_entries = 0;
3405
3406         return ireq;
3407 }
3408
3409 struct isci_request *isci_io_request_from_tag(struct isci_host *ihost,
3410                                               struct sas_task *task,
3411                                               u16 tag)
3412 {
3413         struct isci_request *ireq;
3414
3415         ireq = isci_request_from_tag(ihost, tag);
3416         ireq->ttype_ptr.io_task_ptr = task;
3417         clear_bit(IREQ_TMF, &ireq->flags);
3418         task->lldd_task = ireq;
3419
3420         return ireq;
3421 }
3422
3423 struct isci_request *isci_tmf_request_from_tag(struct isci_host *ihost,
3424                                                struct isci_tmf *isci_tmf,
3425                                                u16 tag)
3426 {
3427         struct isci_request *ireq;
3428
3429         ireq = isci_request_from_tag(ihost, tag);
3430         ireq->ttype_ptr.tmf_task_ptr = isci_tmf;
3431         set_bit(IREQ_TMF, &ireq->flags);
3432
3433         return ireq;
3434 }
3435
3436 int isci_request_execute(struct isci_host *ihost, struct isci_remote_device *idev,
3437                          struct sas_task *task, struct isci_request *ireq)
3438 {
3439         enum sci_status status;
3440         unsigned long flags;
3441         int ret = 0;
3442
3443         status = isci_io_request_build(ihost, ireq, idev);
3444         if (status != SCI_SUCCESS) {
3445                 dev_dbg(&ihost->pdev->dev,
3446                          "%s: request_construct failed - status = 0x%x\n",
3447                          __func__,
3448                          status);
3449                 return status;
3450         }
3451
3452         spin_lock_irqsave(&ihost->scic_lock, flags);
3453
3454         if (test_bit(IDEV_IO_NCQERROR, &idev->flags)) {
3455
3456                 if (isci_task_is_ncq_recovery(task)) {
3457
3458                         /* The device is in an NCQ recovery state.  Issue the
3459                          * request on the task side.  Note that it will
3460                          * complete on the I/O request side because the
3461                          * request was built that way (ie.
3462                          * ireq->is_task_management_request is false).
3463                          */
3464                         status = sci_controller_start_task(ihost,
3465                                                             idev,
3466                                                             ireq);
3467                 } else {
3468                         status = SCI_FAILURE;
3469                 }
3470         } else {
3471                 /* send the request, let the core assign the IO TAG.    */
3472                 status = sci_controller_start_io(ihost, idev,
3473                                                   ireq);
3474         }
3475
3476         if (status != SCI_SUCCESS &&
3477             status != SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED) {
3478                 dev_dbg(&ihost->pdev->dev,
3479                          "%s: failed request start (0x%x)\n",
3480                          __func__, status);
3481                 spin_unlock_irqrestore(&ihost->scic_lock, flags);
3482                 return status;
3483         }
3484         /* Either I/O started OK, or the core has signaled that
3485          * the device needs a target reset.
3486          */
3487         if (status != SCI_SUCCESS) {
3488                 /* The request did not really start in the
3489                  * hardware, so clear the request handle
3490                  * here so no terminations will be done.
3491                  */
3492                 set_bit(IREQ_TERMINATED, &ireq->flags);
3493         }
3494         spin_unlock_irqrestore(&ihost->scic_lock, flags);
3495
3496         if (status ==
3497             SCI_FAILURE_REMOTE_DEVICE_RESET_REQUIRED) {
3498                 /* Signal libsas that we need the SCSI error
3499                  * handler thread to work on this I/O and that
3500                  * we want a device reset.
3501                  */
3502                 spin_lock_irqsave(&task->task_state_lock, flags);
3503                 task->task_state_flags |= SAS_TASK_NEED_DEV_RESET;
3504                 spin_unlock_irqrestore(&task->task_state_lock, flags);
3505
3506                 /* Cause this task to be scheduled in the SCSI error
3507                  * handler thread.
3508                  */
3509                 sas_task_abort(task);
3510
3511                 /* Change the status, since we are holding
3512                  * the I/O until it is managed by the SCSI
3513                  * error handler.
3514                  */
3515                 status = SCI_SUCCESS;
3516         }
3517
3518         return ret;
3519 }