drivers/scsi/qedf/qedf_io.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  QLogic FCoE Offload Driver
   4  *  Copyright (c) 2016-2018 Cavium Inc.
   5  */
   6 #include <linux/spinlock.h>
   7 #include <linux/vmalloc.h>
   8 #include "qedf.h"
   9 #include <scsi/scsi_tcq.h>
  10
  11 void qedf_cmd_timer_set(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
  12         unsigned int timer_msec)
  13 {
  14         queue_delayed_work(qedf->timer_work_queue, &io_req->timeout_work,
  15             msecs_to_jiffies(timer_msec));
  16 }
  17
  18 static void qedf_cmd_timeout(struct work_struct *work)
  19 {
  20
  21         struct qedf_ioreq *io_req =
  22             container_of(work, struct qedf_ioreq, timeout_work.work);
  23         struct qedf_ctx *qedf;
  24         struct qedf_rport *fcport;
  25
  26         if (io_req == NULL) {
  27                 QEDF_INFO(NULL, QEDF_LOG_IO, "io_req is NULL.\n");
  28                 return;
  29         }
  30
  31         fcport = io_req->fcport;
  32         if (io_req->fcport == NULL) {
  33                 QEDF_INFO(NULL, QEDF_LOG_IO,  "fcport is NULL.\n");
  34                 return;
  35         }
  36
  37         qedf = fcport->qedf;
  38
  39         switch (io_req->cmd_type) {
  40         case QEDF_ABTS:
  41                 if (qedf == NULL) {
  42                         QEDF_INFO(NULL, QEDF_LOG_IO,
  43                                   "qedf is NULL for ABTS xid=0x%x.\n",
  44                                   io_req->xid);
  45                         return;
  46                 }
  47
  48                 QEDF_ERR((&qedf->dbg_ctx), "ABTS timeout, xid=0x%x.\n",
  49                     io_req->xid);
  50                 /* Cleanup timed out ABTS */
  51                 qedf_initiate_cleanup(io_req, true);
  52                 complete(&io_req->abts_done);
  53
  54                 /*
  55                  * Need to call kref_put for reference taken when initiate_abts
  56                  * was called since abts_compl won't be called now that we've
  57                  * cleaned up the task.
  58                  */
  59                 kref_put(&io_req->refcount, qedf_release_cmd);
  60
  61                 /* Clear in abort bit now that we're done with the command */
  62                 clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
  63
  64                 /*
  65                  * Now that the original I/O and the ABTS are complete see
  66                  * if we need to reconnect to the target.
  67                  */
  68                 qedf_restart_rport(fcport);
  69                 break;
  70         case QEDF_ELS:
  71                 if (!qedf) {
  72                         QEDF_INFO(NULL, QEDF_LOG_IO,
  73                                   "qedf is NULL for ELS xid=0x%x.\n",
  74                                   io_req->xid);
  75                         return;
  76                 }
  77                 /* ELS request no longer outstanding since it timed out */
  78                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
  79
  80                 kref_get(&io_req->refcount);
  81                 /*
  82                  * Don't attempt to clean an ELS timeout as any subseqeunt
  83                  * ABTS or cleanup requests just hang.  For now just free
  84                  * the resources of the original I/O and the RRQ
  85                  */
  86                 QEDF_ERR(&(qedf->dbg_ctx), "ELS timeout, xid=0x%x.\n",
  87                           io_req->xid);
  88                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
  89                 /* Call callback function to complete command */
  90                 if (io_req->cb_func && io_req->cb_arg) {
  91                         io_req->cb_func(io_req->cb_arg);
  92                         io_req->cb_arg = NULL;
  93                 }
  94                 qedf_initiate_cleanup(io_req, true);
  95                 kref_put(&io_req->refcount, qedf_release_cmd);
  96                 break;
  97         case QEDF_SEQ_CLEANUP:
  98                 QEDF_ERR(&(qedf->dbg_ctx), "Sequence cleanup timeout, "
  99                     "xid=0x%x.\n", io_req->xid);
 100                 qedf_initiate_cleanup(io_req, true);
 101                 io_req->event = QEDF_IOREQ_EV_ELS_TMO;
 102                 qedf_process_seq_cleanup_compl(qedf, NULL, io_req);
 103                 break;
 104         default:
 105                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 106                           "Hit default case, xid=0x%x.\n", io_req->xid);
 107                 break;
 108         }
 109 }
 110
 111 void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr)
 112 {
 113         struct io_bdt *bdt_info;
 114         struct qedf_ctx *qedf = cmgr->qedf;
 115         size_t bd_tbl_sz;
 116         u16 min_xid = 0;
 117         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 118         int num_ios;
 119         int i;
 120         struct qedf_ioreq *io_req;
 121
 122         num_ios = max_xid - min_xid + 1;
 123
 124         /* Free fcoe_bdt_ctx structures */
 125         if (!cmgr->io_bdt_pool) {
 126                 QEDF_ERR(&qedf->dbg_ctx, "io_bdt_pool is NULL.\n");
 127                 goto free_cmd_pool;
 128         }
 129
 130         bd_tbl_sz = QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge);
 131         for (i = 0; i < num_ios; i++) {
 132                 bdt_info = cmgr->io_bdt_pool[i];
 133                 if (bdt_info->bd_tbl) {
 134                         dma_free_coherent(&qedf->pdev->dev, bd_tbl_sz,
 135                             bdt_info->bd_tbl, bdt_info->bd_tbl_dma);
 136                         bdt_info->bd_tbl = NULL;
 137                 }
 138         }
 139
 140         /* Destroy io_bdt pool */
 141         for (i = 0; i < num_ios; i++) {
 142                 kfree(cmgr->io_bdt_pool[i]);
 143                 cmgr->io_bdt_pool[i] = NULL;
 144         }
 145
 146         kfree(cmgr->io_bdt_pool);
 147         cmgr->io_bdt_pool = NULL;
 148
 149 free_cmd_pool:
 150
 151         for (i = 0; i < num_ios; i++) {
 152                 io_req = &cmgr->cmds[i];
 153                 kfree(io_req->sgl_task_params);
 154                 kfree(io_req->task_params);
 155                 /* Make sure we free per command sense buffer */
 156                 if (io_req->sense_buffer)
 157                         dma_free_coherent(&qedf->pdev->dev,
 158                             QEDF_SCSI_SENSE_BUFFERSIZE, io_req->sense_buffer,
 159                             io_req->sense_buffer_dma);
 160                 cancel_delayed_work_sync(&io_req->rrq_work);
 161         }
 162
 163         /* Free command manager itself */
 164         vfree(cmgr);
 165 }
 166
 167 static void qedf_handle_rrq(struct work_struct *work)
 168 {
 169         struct qedf_ioreq *io_req =
 170             container_of(work, struct qedf_ioreq, rrq_work.work);
 171
 172         atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_ACTIVE);
 173         qedf_send_rrq(io_req);
 174
 175 }
 176
 177 struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
 178 {
 179         struct qedf_cmd_mgr *cmgr;
 180         struct io_bdt *bdt_info;
 181         struct qedf_ioreq *io_req;
 182         u16 xid;
 183         int i;
 184         int num_ios;
 185         u16 min_xid = 0;
 186         u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 187
 188         /* Make sure num_queues is already set before calling this function */
 189         if (!qedf->num_queues) {
 190                 QEDF_ERR(&(qedf->dbg_ctx), "num_queues is not set.\n");
 191                 return NULL;
 192         }
 193
 194         if (max_xid <= min_xid || max_xid == FC_XID_UNKNOWN) {
 195                 QEDF_WARN(&(qedf->dbg_ctx), "Invalid min_xid 0x%x and "
 196                            "max_xid 0x%x.\n", min_xid, max_xid);
 197                 return NULL;
 198         }
 199
 200         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "min xid 0x%x, max xid "
 201                    "0x%x.\n", min_xid, max_xid);
 202
 203         num_ios = max_xid - min_xid + 1;
 204
 205         cmgr = vzalloc(sizeof(struct qedf_cmd_mgr));
 206         if (!cmgr) {
 207                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc cmd mgr.\n");
 208                 return NULL;
 209         }
 210
 211         cmgr->qedf = qedf;
 212         spin_lock_init(&cmgr->lock);
 213
 214         /*
 215          * Initialize I/O request fields.
 216          */
 217         xid = 0;
 218
 219         for (i = 0; i < num_ios; i++) {
 220                 io_req = &cmgr->cmds[i];
 221                 INIT_DELAYED_WORK(&io_req->timeout_work, qedf_cmd_timeout);
 222
 223                 io_req->xid = xid++;
 224
 225                 INIT_DELAYED_WORK(&io_req->rrq_work, qedf_handle_rrq);
 226
 227                 /* Allocate DMA memory to hold sense buffer */
 228                 io_req->sense_buffer = dma_alloc_coherent(&qedf->pdev->dev,
 229                     QEDF_SCSI_SENSE_BUFFERSIZE, &io_req->sense_buffer_dma,
 230                     GFP_KERNEL);
 231                 if (!io_req->sense_buffer) {
 232                         QEDF_ERR(&qedf->dbg_ctx,
 233                                  "Failed to alloc sense buffer.\n");
 234                         goto mem_err;
 235                 }
 236
 237                 /* Allocate task parameters to pass to f/w init funcions */
 238                 io_req->task_params = kzalloc(sizeof(*io_req->task_params),
 239                                               GFP_KERNEL);
 240                 if (!io_req->task_params) {
 241                         QEDF_ERR(&(qedf->dbg_ctx),
 242                                  "Failed to allocate task_params for xid=0x%x\n",
 243                                  i);
 244                         goto mem_err;
 245                 }
 246
 247                 /*
 248                  * Allocate scatter/gather list info to pass to f/w init
 249                  * functions.
 250                  */
 251                 io_req->sgl_task_params = kzalloc(
 252                     sizeof(struct scsi_sgl_task_params), GFP_KERNEL);
 253                 if (!io_req->sgl_task_params) {
 254                         QEDF_ERR(&(qedf->dbg_ctx),
 255                                  "Failed to allocate sgl_task_params for xid=0x%x\n",
 256                                  i);
 257                         goto mem_err;
 258                 }
 259         }
 260
 261         /* Allocate pool of io_bdts - one for each qedf_ioreq */
 262         cmgr->io_bdt_pool = kmalloc_array(num_ios, sizeof(struct io_bdt *),
 263             GFP_KERNEL);
 264
 265         if (!cmgr->io_bdt_pool) {
 266                 QEDF_WARN(&(qedf->dbg_ctx), "Failed to alloc io_bdt_pool.\n");
 267                 goto mem_err;
 268         }
 269
 270         for (i = 0; i < num_ios; i++) {
 271                 cmgr->io_bdt_pool[i] = kmalloc(sizeof(struct io_bdt),
 272                     GFP_KERNEL);
 273                 if (!cmgr->io_bdt_pool[i]) {
 274                         QEDF_WARN(&(qedf->dbg_ctx),
 275                                   "Failed to alloc io_bdt_pool[%d].\n", i);
 276                         goto mem_err;
 277                 }
 278         }
 279
 280         for (i = 0; i < num_ios; i++) {
 281                 bdt_info = cmgr->io_bdt_pool[i];
 282                 bdt_info->bd_tbl = dma_alloc_coherent(&qedf->pdev->dev,
 283                     QEDF_MAX_BDS_PER_CMD * sizeof(struct scsi_sge),
 284                     &bdt_info->bd_tbl_dma, GFP_KERNEL);
 285                 if (!bdt_info->bd_tbl) {
 286                         QEDF_WARN(&(qedf->dbg_ctx),
 287                                   "Failed to alloc bdt_tbl[%d].\n", i);
 288                         goto mem_err;
 289                 }
 290         }
 291         atomic_set(&cmgr->free_list_cnt, num_ios);
 292         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 293             "cmgr->free_list_cnt=%d.\n",
 294             atomic_read(&cmgr->free_list_cnt));
 295
 296         return cmgr;
 297
 298 mem_err:
 299         qedf_cmd_mgr_free(cmgr);
 300         return NULL;
 301 }
 302
 303 struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
 304 {
 305         struct qedf_ctx *qedf = fcport->qedf;
 306         struct qedf_cmd_mgr *cmd_mgr = qedf->cmd_mgr;
 307         struct qedf_ioreq *io_req = NULL;
 308         struct io_bdt *bd_tbl;
 309         u16 xid;
 310         uint32_t free_sqes;
 311         int i;
 312         unsigned long flags;
 313
 314         free_sqes = atomic_read(&fcport->free_sqes);
 315
 316         if (!free_sqes) {
 317                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 318                     "Returning NULL, free_sqes=%d.\n ",
 319                     free_sqes);
 320                 goto out_failed;
 321         }
 322
 323         /* Limit the number of outstanding R/W tasks */
 324         if ((atomic_read(&fcport->num_active_ios) >=
 325             NUM_RW_TASKS_PER_CONNECTION)) {
 326                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 327                     "Returning NULL, num_active_ios=%d.\n",
 328                     atomic_read(&fcport->num_active_ios));
 329                 goto out_failed;
 330         }
 331
 332         /* Limit global TIDs certain tasks */
 333         if (atomic_read(&cmd_mgr->free_list_cnt) <= GBL_RSVD_TASKS) {
 334                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 335                     "Returning NULL, free_list_cnt=%d.\n",
 336                     atomic_read(&cmd_mgr->free_list_cnt));
 337                 goto out_failed;
 338         }
 339
 340         spin_lock_irqsave(&cmd_mgr->lock, flags);
 341         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
 342                 io_req = &cmd_mgr->cmds[cmd_mgr->idx];
 343                 cmd_mgr->idx++;
 344                 if (cmd_mgr->idx == FCOE_PARAMS_NUM_TASKS)
 345                         cmd_mgr->idx = 0;
 346
 347                 /* Check to make sure command was previously freed */
 348                 if (!io_req->alloc)
 349                         break;
 350         }
 351
 352         if (i == FCOE_PARAMS_NUM_TASKS) {
 353                 spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 354                 goto out_failed;
 355         }
 356
 357         if (test_bit(QEDF_CMD_DIRTY, &io_req->flags))
 358                 QEDF_ERR(&qedf->dbg_ctx,
 359                          "io_req found to be dirty ox_id = 0x%x.\n",
 360                          io_req->xid);
 361
 362         /* Clear any flags now that we've reallocated the xid */
 363         io_req->flags = 0;
 364         io_req->alloc = 1;
 365         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 366
 367         atomic_inc(&fcport->num_active_ios);
 368         atomic_dec(&fcport->free_sqes);
 369         xid = io_req->xid;
 370         atomic_dec(&cmd_mgr->free_list_cnt);
 371
 372         io_req->cmd_mgr = cmd_mgr;
 373         io_req->fcport = fcport;
 374
 375         /* Clear any stale sc_cmd back pointer */
 376         io_req->sc_cmd = NULL;
 377         io_req->lun = -1;
 378
 379         /* Hold the io_req against deletion */
 380         kref_init(&io_req->refcount);   /* ID: 001 */
 381         atomic_set(&io_req->state, QEDFC_CMD_ST_IO_ACTIVE);
 382
 383         /* Bind io_bdt for this io_req */
 384         /* Have a static link between io_req and io_bdt_pool */
 385         bd_tbl = io_req->bd_tbl = cmd_mgr->io_bdt_pool[xid];
 386         if (bd_tbl == NULL) {
 387                 QEDF_ERR(&(qedf->dbg_ctx), "bd_tbl is NULL, xid=%x.\n", xid);
 388                 kref_put(&io_req->refcount, qedf_release_cmd);
 389                 goto out_failed;
 390         }
 391         bd_tbl->io_req = io_req;
 392         io_req->cmd_type = cmd_type;
 393         io_req->tm_flags = 0;
 394
 395         /* Reset sequence offset data */
 396         io_req->rx_buf_off = 0;
 397         io_req->tx_buf_off = 0;
 398         io_req->rx_id = 0xffff; /* No OX_ID */
 399
 400         return io_req;
 401
 402 out_failed:
 403         /* Record failure for stats and return NULL to caller */
 404         qedf->alloc_failures++;
 405         return NULL;
 406 }
 407
 408 static void qedf_free_mp_resc(struct qedf_ioreq *io_req)
 409 {
 410         struct qedf_mp_req *mp_req = &(io_req->mp_req);
 411         struct qedf_ctx *qedf = io_req->fcport->qedf;
 412         uint64_t sz = sizeof(struct scsi_sge);
 413
 414         /* clear tm flags */
 415         if (mp_req->mp_req_bd) {
 416                 dma_free_coherent(&qedf->pdev->dev, sz,
 417                     mp_req->mp_req_bd, mp_req->mp_req_bd_dma);
 418                 mp_req->mp_req_bd = NULL;
 419         }
 420         if (mp_req->mp_resp_bd) {
 421                 dma_free_coherent(&qedf->pdev->dev, sz,
 422                     mp_req->mp_resp_bd, mp_req->mp_resp_bd_dma);
 423                 mp_req->mp_resp_bd = NULL;
 424         }
 425         if (mp_req->req_buf) {
 426                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
 427                     mp_req->req_buf, mp_req->req_buf_dma);
 428                 mp_req->req_buf = NULL;
 429         }
 430         if (mp_req->resp_buf) {
 431                 dma_free_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
 432                     mp_req->resp_buf, mp_req->resp_buf_dma);
 433                 mp_req->resp_buf = NULL;
 434         }
 435 }
 436
 437 void qedf_release_cmd(struct kref *ref)
 438 {
 439         struct qedf_ioreq *io_req =
 440             container_of(ref, struct qedf_ioreq, refcount);
 441         struct qedf_cmd_mgr *cmd_mgr = io_req->cmd_mgr;
 442         struct qedf_rport *fcport = io_req->fcport;
 443         unsigned long flags;
 444
 445         if (io_req->cmd_type == QEDF_SCSI_CMD) {
 446                 QEDF_WARN(&fcport->qedf->dbg_ctx,
 447                           "Cmd released called without scsi_done called, io_req %p xid=0x%x.\n",
 448                           io_req, io_req->xid);
 449                 WARN_ON(io_req->sc_cmd);
 450         }
 451
 452         if (io_req->cmd_type == QEDF_ELS ||
 453             io_req->cmd_type == QEDF_TASK_MGMT_CMD)
 454                 qedf_free_mp_resc(io_req);
 455
 456         atomic_inc(&cmd_mgr->free_list_cnt);
 457         atomic_dec(&fcport->num_active_ios);
 458         atomic_set(&io_req->state, QEDF_CMD_ST_INACTIVE);
 459         if (atomic_read(&fcport->num_active_ios) < 0) {
 460                 QEDF_WARN(&(fcport->qedf->dbg_ctx), "active_ios < 0.\n");
 461                 WARN_ON(1);
 462         }
 463
 464         /* Increment task retry identifier now that the request is released */
 465         io_req->task_retry_identifier++;
 466         io_req->fcport = NULL;
 467
 468         clear_bit(QEDF_CMD_DIRTY, &io_req->flags);
 469         io_req->cpu = 0;
 470         spin_lock_irqsave(&cmd_mgr->lock, flags);
 471         io_req->fcport = NULL;
 472         io_req->alloc = 0;
 473         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 474 }
 475
 476 static int qedf_map_sg(struct qedf_ioreq *io_req)
 477 {
 478         struct scsi_cmnd *sc = io_req->sc_cmd;
 479         struct Scsi_Host *host = sc->device->host;
 480         struct fc_lport *lport = shost_priv(host);
 481         struct qedf_ctx *qedf = lport_priv(lport);
 482         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
 483         struct scatterlist *sg;
 484         int byte_count = 0;
 485         int sg_count = 0;
 486         int bd_count = 0;
 487         u32 sg_len;
 488         u64 addr;
 489         int i = 0;
 490
 491         sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
 492             scsi_sg_count(sc), sc->sc_data_direction);
 493         sg = scsi_sglist(sc);
 494
 495         io_req->sge_type = QEDF_IOREQ_UNKNOWN_SGE;
 496
 497         if (sg_count <= 8 || io_req->io_req_flags == QEDF_READ)
 498                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 499
 500         scsi_for_each_sg(sc, sg, sg_count, i) {
 501                 sg_len = (u32)sg_dma_len(sg);
 502                 addr = (u64)sg_dma_address(sg);
 503
 504                 /*
 505                  * Intermediate s/g element so check if start address
 506                  * is page aligned.  Only required for writes and only if the
 507                  * number of scatter/gather elements is 8 or more.
 508                  */
 509                 if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE && (i) &&
 510                     (i != (sg_count - 1)) && sg_len < QEDF_PAGE_SIZE)
 511                         io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
 512
 513                 bd[bd_count].sge_addr.lo = cpu_to_le32(U64_LO(addr));
 514                 bd[bd_count].sge_addr.hi  = cpu_to_le32(U64_HI(addr));
 515                 bd[bd_count].sge_len = cpu_to_le32(sg_len);
 516
 517                 bd_count++;
 518                 byte_count += sg_len;
 519         }
 520
 521         /* To catch a case where FAST and SLOW nothing is set, set FAST */
 522         if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE)
 523                 io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 524
 525         if (byte_count != scsi_bufflen(sc))
 526                 QEDF_ERR(&(qedf->dbg_ctx), "byte_count = %d != "
 527                           "scsi_bufflen = %d, task_id = 0x%x.\n", byte_count,
 528                            scsi_bufflen(sc), io_req->xid);
 529
 530         return bd_count;
 531 }
 532
 533 static int qedf_build_bd_list_from_sg(struct qedf_ioreq *io_req)
 534 {
 535         struct scsi_cmnd *sc = io_req->sc_cmd;
 536         struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
 537         int bd_count;
 538
 539         if (scsi_sg_count(sc)) {
 540                 bd_count = qedf_map_sg(io_req);
 541                 if (bd_count == 0)
 542                         return -ENOMEM;
 543         } else {
 544                 bd_count = 0;
 545                 bd[0].sge_addr.lo = bd[0].sge_addr.hi = 0;
 546                 bd[0].sge_len = 0;
 547         }
 548         io_req->bd_tbl->bd_valid = bd_count;
 549
 550         return 0;
 551 }
 552
 553 static void qedf_build_fcp_cmnd(struct qedf_ioreq *io_req,
 554                                   struct fcp_cmnd *fcp_cmnd)
 555 {
 556         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 557
 558         /* fcp_cmnd is 32 bytes */
 559         memset(fcp_cmnd, 0, FCP_CMND_LEN);
 560
 561         /* 8 bytes: SCSI LUN info */
 562         int_to_scsilun(sc_cmd->device->lun,
 563                         (struct scsi_lun *)&fcp_cmnd->fc_lun);
 564
 565         /* 4 bytes: flag info */
 566         fcp_cmnd->fc_pri_ta = 0;
 567         fcp_cmnd->fc_tm_flags = io_req->tm_flags;
 568         fcp_cmnd->fc_flags = io_req->io_req_flags;
 569         fcp_cmnd->fc_cmdref = 0;
 570
 571         /* Populate data direction */
 572         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
 573                 fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
 574         } else {
 575                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE)
 576                         fcp_cmnd->fc_flags |= FCP_CFL_WRDATA;
 577                 else if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE)
 578                         fcp_cmnd->fc_flags |= FCP_CFL_RDDATA;
 579         }
 580
 581         fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
 582
 583         /* 16 bytes: CDB information */
 584         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD)
 585                 memcpy(fcp_cmnd->fc_cdb, sc_cmd->cmnd, sc_cmd->cmd_len);
 586
 587         /* 4 bytes: FCP data length */
 588         fcp_cmnd->fc_dl = htonl(io_req->data_xfer_len);
 589 }
 590
 591 static void  qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
 592         struct qedf_ioreq *io_req, struct e4_fcoe_task_context *task_ctx,
 593         struct fcoe_wqe *sqe)
 594 {
 595         enum fcoe_task_type task_type;
 596         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 597         struct io_bdt *bd_tbl = io_req->bd_tbl;
 598         u8 fcp_cmnd[32];
 599         u32 tmp_fcp_cmnd[8];
 600         int bd_count = 0;
 601         struct qedf_ctx *qedf = fcport->qedf;
 602         uint16_t cq_idx = smp_processor_id() % qedf->num_queues;
 603         struct regpair sense_data_buffer_phys_addr;
 604         u32 tx_io_size = 0;
 605         u32 rx_io_size = 0;
 606         int i, cnt;
 607
 608         /* Note init_initiator_rw_fcoe_task memsets the task context */
 609         io_req->task = task_ctx;
 610         memset(task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 611         memset(io_req->task_params, 0, sizeof(struct fcoe_task_params));
 612         memset(io_req->sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 613
 614         /* Set task type bassed on DMA directio of command */
 615         if (io_req->cmd_type == QEDF_TASK_MGMT_CMD) {
 616                 task_type = FCOE_TASK_TYPE_READ_INITIATOR;
 617         } else {
 618                 if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
 619                         task_type = FCOE_TASK_TYPE_WRITE_INITIATOR;
 620                         tx_io_size = io_req->data_xfer_len;
 621                 } else {
 622                         task_type = FCOE_TASK_TYPE_READ_INITIATOR;
 623                         rx_io_size = io_req->data_xfer_len;
 624                 }
 625         }
 626
 627         /* Setup the fields for fcoe_task_params */
 628         io_req->task_params->context = task_ctx;
 629         io_req->task_params->sqe = sqe;
 630         io_req->task_params->task_type = task_type;
 631         io_req->task_params->tx_io_size = tx_io_size;
 632         io_req->task_params->rx_io_size = rx_io_size;
 633         io_req->task_params->conn_cid = fcport->fw_cid;
 634         io_req->task_params->itid = io_req->xid;
 635         io_req->task_params->cq_rss_number = cq_idx;
 636         io_req->task_params->is_tape_device = fcport->dev_type;
 637
 638         /* Fill in information for scatter/gather list */
 639         if (io_req->cmd_type != QEDF_TASK_MGMT_CMD) {
 640                 bd_count = bd_tbl->bd_valid;
 641                 io_req->sgl_task_params->sgl = bd_tbl->bd_tbl;
 642                 io_req->sgl_task_params->sgl_phys_addr.lo =
 643                         U64_LO(bd_tbl->bd_tbl_dma);
 644                 io_req->sgl_task_params->sgl_phys_addr.hi =
 645                         U64_HI(bd_tbl->bd_tbl_dma);
 646                 io_req->sgl_task_params->num_sges = bd_count;
 647                 io_req->sgl_task_params->total_buffer_size =
 648                     scsi_bufflen(io_req->sc_cmd);
 649                 if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 650                         io_req->sgl_task_params->small_mid_sge = 1;
 651                 else
 652                         io_req->sgl_task_params->small_mid_sge = 0;
 653         }
 654
 655         /* Fill in physical address of sense buffer */
 656         sense_data_buffer_phys_addr.lo = U64_LO(io_req->sense_buffer_dma);
 657         sense_data_buffer_phys_addr.hi = U64_HI(io_req->sense_buffer_dma);
 658
 659         /* fill FCP_CMND IU */
 660         qedf_build_fcp_cmnd(io_req, (struct fcp_cmnd *)tmp_fcp_cmnd);
 661
 662         /* Swap fcp_cmnd since FC is big endian */
 663         cnt = sizeof(struct fcp_cmnd) / sizeof(u32);
 664         for (i = 0; i < cnt; i++) {
 665                 tmp_fcp_cmnd[i] = cpu_to_be32(tmp_fcp_cmnd[i]);
 666         }
 667         memcpy(fcp_cmnd, tmp_fcp_cmnd, sizeof(struct fcp_cmnd));
 668
 669         init_initiator_rw_fcoe_task(io_req->task_params,
 670                                     io_req->sgl_task_params,
 671                                     sense_data_buffer_phys_addr,
 672                                     io_req->task_retry_identifier, fcp_cmnd);
 673
 674         /* Increment SGL type counters */
 675         if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 676                 qedf->slow_sge_ios++;
 677         else
 678                 qedf->fast_sge_ios++;
 679 }
 680
 681 void qedf_init_mp_task(struct qedf_ioreq *io_req,
 682         struct e4_fcoe_task_context *task_ctx, struct fcoe_wqe *sqe)
 683 {
 684         struct qedf_mp_req *mp_req = &(io_req->mp_req);
 685         struct qedf_rport *fcport = io_req->fcport;
 686         struct qedf_ctx *qedf = io_req->fcport->qedf;
 687         struct fc_frame_header *fc_hdr;
 688         struct fcoe_tx_mid_path_params task_fc_hdr;
 689         struct scsi_sgl_task_params tx_sgl_task_params;
 690         struct scsi_sgl_task_params rx_sgl_task_params;
 691
 692         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 693                   "Initializing MP task for cmd_type=%d\n",
 694                   io_req->cmd_type);
 695
 696         qedf->control_requests++;
 697
 698         memset(&tx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 699         memset(&rx_sgl_task_params, 0, sizeof(struct scsi_sgl_task_params));
 700         memset(task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 701         memset(&task_fc_hdr, 0, sizeof(struct fcoe_tx_mid_path_params));
 702
 703         /* Setup the task from io_req for easy reference */
 704         io_req->task = task_ctx;
 705
 706         /* Setup the fields for fcoe_task_params */
 707         io_req->task_params->context = task_ctx;
 708         io_req->task_params->sqe = sqe;
 709         io_req->task_params->task_type = FCOE_TASK_TYPE_MIDPATH;
 710         io_req->task_params->tx_io_size = io_req->data_xfer_len;
 711         /* rx_io_size tells the f/w how large a response buffer we have */
 712         io_req->task_params->rx_io_size = PAGE_SIZE;
 713         io_req->task_params->conn_cid = fcport->fw_cid;
 714         io_req->task_params->itid = io_req->xid;
 715         /* Return middle path commands on CQ 0 */
 716         io_req->task_params->cq_rss_number = 0;
 717         io_req->task_params->is_tape_device = fcport->dev_type;
 718
 719         fc_hdr = &(mp_req->req_fc_hdr);
 720         /* Set OX_ID and RX_ID based on driver task id */
 721         fc_hdr->fh_ox_id = io_req->xid;
 722         fc_hdr->fh_rx_id = htons(0xffff);
 723
 724         /* Set up FC header information */
 725         task_fc_hdr.parameter = fc_hdr->fh_parm_offset;
 726         task_fc_hdr.r_ctl = fc_hdr->fh_r_ctl;
 727         task_fc_hdr.type = fc_hdr->fh_type;
 728         task_fc_hdr.cs_ctl = fc_hdr->fh_cs_ctl;
 729         task_fc_hdr.df_ctl = fc_hdr->fh_df_ctl;
 730         task_fc_hdr.rx_id = fc_hdr->fh_rx_id;
 731         task_fc_hdr.ox_id = fc_hdr->fh_ox_id;
 732
 733         /* Set up s/g list parameters for request buffer */
 734         tx_sgl_task_params.sgl = mp_req->mp_req_bd;
 735         tx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_req_bd_dma);
 736         tx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_req_bd_dma);
 737         tx_sgl_task_params.num_sges = 1;
 738         /* Set PAGE_SIZE for now since sg element is that size ??? */
 739         tx_sgl_task_params.total_buffer_size = io_req->data_xfer_len;
 740         tx_sgl_task_params.small_mid_sge = 0;
 741
 742         /* Set up s/g list parameters for request buffer */
 743         rx_sgl_task_params.sgl = mp_req->mp_resp_bd;
 744         rx_sgl_task_params.sgl_phys_addr.lo = U64_LO(mp_req->mp_resp_bd_dma);
 745         rx_sgl_task_params.sgl_phys_addr.hi = U64_HI(mp_req->mp_resp_bd_dma);
 746         rx_sgl_task_params.num_sges = 1;
 747         /* Set PAGE_SIZE for now since sg element is that size ??? */
 748         rx_sgl_task_params.total_buffer_size = PAGE_SIZE;
 749         rx_sgl_task_params.small_mid_sge = 0;
 750
 751
 752         /*
 753          * Last arg is 0 as previous code did not set that we wanted the
 754          * fc header information.
 755          */
 756         init_initiator_midpath_unsolicited_fcoe_task(io_req->task_params,
 757                                                      &task_fc_hdr,
 758                                                      &tx_sgl_task_params,
 759                                                      &rx_sgl_task_params, 0);
 760 }
 761
 762 /* Presumed that fcport->rport_lock is held */
 763 u16 qedf_get_sqe_idx(struct qedf_rport *fcport)
 764 {
 765         uint16_t total_sqe = (fcport->sq_mem_size)/(sizeof(struct fcoe_wqe));
 766         u16 rval;
 767
 768         rval = fcport->sq_prod_idx;
 769
 770         /* Adjust ring index */
 771         fcport->sq_prod_idx++;
 772         fcport->fw_sq_prod_idx++;
 773         if (fcport->sq_prod_idx == total_sqe)
 774                 fcport->sq_prod_idx = 0;
 775
 776         return rval;
 777 }
 778
 779 void qedf_ring_doorbell(struct qedf_rport *fcport)
 780 {
 781         struct fcoe_db_data dbell = { 0 };
 782
 783         dbell.agg_flags = 0;
 784
 785         dbell.params |= DB_DEST_XCM << FCOE_DB_DATA_DEST_SHIFT;
 786         dbell.params |= DB_AGG_CMD_SET << FCOE_DB_DATA_AGG_CMD_SHIFT;
 787         dbell.params |= DQ_XCM_FCOE_SQ_PROD_CMD <<
 788             FCOE_DB_DATA_AGG_VAL_SEL_SHIFT;
 789
 790         dbell.sq_prod = fcport->fw_sq_prod_idx;
 791         /* wmb makes sure that the BDs data is updated before updating the
 792          * producer, otherwise FW may read old data from the BDs.
 793          */
 794         wmb();
 795         barrier();
 796         writel(*(u32 *)&dbell, fcport->p_doorbell);
 797         /*
 798          * Fence required to flush the write combined buffer, since another
 799          * CPU may write to the same doorbell address and data may be lost
 800          * due to relaxed order nature of write combined bar.
 801          */
 802         wmb();
 803 }
 804
 805 static void qedf_trace_io(struct qedf_rport *fcport, struct qedf_ioreq *io_req,
 806                           int8_t direction)
 807 {
 808         struct qedf_ctx *qedf = fcport->qedf;
 809         struct qedf_io_log *io_log;
 810         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 811         unsigned long flags;
 812         uint8_t op;
 813
 814         spin_lock_irqsave(&qedf->io_trace_lock, flags);
 815
 816         io_log = &qedf->io_trace_buf[qedf->io_trace_idx];
 817         io_log->direction = direction;
 818         io_log->task_id = io_req->xid;
 819         io_log->port_id = fcport->rdata->ids.port_id;
 820         io_log->lun = sc_cmd->device->lun;
 821         io_log->op = op = sc_cmd->cmnd[0];
 822         io_log->lba[0] = sc_cmd->cmnd[2];
 823         io_log->lba[1] = sc_cmd->cmnd[3];
 824         io_log->lba[2] = sc_cmd->cmnd[4];
 825         io_log->lba[3] = sc_cmd->cmnd[5];
 826         io_log->bufflen = scsi_bufflen(sc_cmd);
 827         io_log->sg_count = scsi_sg_count(sc_cmd);
 828         io_log->result = sc_cmd->result;
 829         io_log->jiffies = jiffies;
 830         io_log->refcount = kref_read(&io_req->refcount);
 831
 832         if (direction == QEDF_IO_TRACE_REQ) {
 833                 /* For requests we only care abot the submission CPU */
 834                 io_log->req_cpu = io_req->cpu;
 835                 io_log->int_cpu = 0;
 836                 io_log->rsp_cpu = 0;
 837         } else if (direction == QEDF_IO_TRACE_RSP) {
 838                 io_log->req_cpu = io_req->cpu;
 839                 io_log->int_cpu = io_req->int_cpu;
 840                 io_log->rsp_cpu = smp_processor_id();
 841         }
 842
 843         io_log->sge_type = io_req->sge_type;
 844
 845         qedf->io_trace_idx++;
 846         if (qedf->io_trace_idx == QEDF_IO_TRACE_SIZE)
 847                 qedf->io_trace_idx = 0;
 848
 849         spin_unlock_irqrestore(&qedf->io_trace_lock, flags);
 850 }
 851
 852 int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 853 {
 854         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
 855         struct Scsi_Host *host = sc_cmd->device->host;
 856         struct fc_lport *lport = shost_priv(host);
 857         struct qedf_ctx *qedf = lport_priv(lport);
 858         struct e4_fcoe_task_context *task_ctx;
 859         u16 xid;
 860         struct fcoe_wqe *sqe;
 861         u16 sqe_idx;
 862
 863         /* Initialize rest of io_req fileds */
 864         io_req->data_xfer_len = scsi_bufflen(sc_cmd);
 865         sc_cmd->SCp.ptr = (char *)io_req;
 866         io_req->sge_type = QEDF_IOREQ_FAST_SGE; /* Assume fast SGL by default */
 867
 868         /* Record which cpu this request is associated with */
 869         io_req->cpu = smp_processor_id();
 870
 871         if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
 872                 io_req->io_req_flags = QEDF_READ;
 873                 qedf->input_requests++;
 874         } else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
 875                 io_req->io_req_flags = QEDF_WRITE;
 876                 qedf->output_requests++;
 877         } else {
 878                 io_req->io_req_flags = 0;
 879                 qedf->control_requests++;
 880         }
 881
 882         xid = io_req->xid;
 883
 884         /* Build buffer descriptor list for firmware from sg list */
 885         if (qedf_build_bd_list_from_sg(io_req)) {
 886                 QEDF_ERR(&(qedf->dbg_ctx), "BD list creation failed.\n");
 887                 /* Release cmd will release io_req, but sc_cmd is assigned */
 888                 io_req->sc_cmd = NULL;
 889                 kref_put(&io_req->refcount, qedf_release_cmd);
 890                 return -EAGAIN;
 891         }
 892
 893         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
 894             test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 895                 QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
 896                 /* Release cmd will release io_req, but sc_cmd is assigned */
 897                 io_req->sc_cmd = NULL;
 898                 kref_put(&io_req->refcount, qedf_release_cmd);
 899                 return -EINVAL;
 900         }
 901
 902         /* Record LUN number for later use if we neeed them */
 903         io_req->lun = (int)sc_cmd->device->lun;
 904
 905         /* Obtain free SQE */
 906         sqe_idx = qedf_get_sqe_idx(fcport);
 907         sqe = &fcport->sq[sqe_idx];
 908         memset(sqe, 0, sizeof(struct fcoe_wqe));
 909
 910         /* Get the task context */
 911         task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
 912         if (!task_ctx) {
 913                 QEDF_WARN(&(qedf->dbg_ctx), "task_ctx is NULL, xid=%d.\n",
 914                            xid);
 915                 /* Release cmd will release io_req, but sc_cmd is assigned */
 916                 io_req->sc_cmd = NULL;
 917                 kref_put(&io_req->refcount, qedf_release_cmd);
 918                 return -EINVAL;
 919         }
 920
 921         qedf_init_task(fcport, lport, io_req, task_ctx, sqe);
 922
 923         /* Ring doorbell */
 924         qedf_ring_doorbell(fcport);
 925
 926         /* Set that command is with the firmware now */
 927         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
 928
 929         if (qedf_io_tracing && io_req->sc_cmd)
 930                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_REQ);
 931
 932         return false;
 933 }
 934
 935 int
 936 qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 937 {
 938         struct fc_lport *lport = shost_priv(host);
 939         struct qedf_ctx *qedf = lport_priv(lport);
 940         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
 941         struct fc_rport_libfc_priv *rp = rport->dd_data;
 942         struct qedf_rport *fcport;
 943         struct qedf_ioreq *io_req;
 944         int rc = 0;
 945         int rval;
 946         unsigned long flags = 0;
 947         int num_sgs = 0;
 948
 949         num_sgs = scsi_sg_count(sc_cmd);
 950         if (scsi_sg_count(sc_cmd) > QEDF_MAX_BDS_PER_CMD) {
 951                 QEDF_ERR(&qedf->dbg_ctx,
 952                          "Number of SG elements %d exceeds what hardware limitation of %d.\n",
 953                          num_sgs, QEDF_MAX_BDS_PER_CMD);
 954                 sc_cmd->result = DID_ERROR;
 955                 sc_cmd->scsi_done(sc_cmd);
 956                 return 0;
 957         }
 958
 959         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
 960             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
 961                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 962                           "Returning DNC as unloading or stop io, flags 0x%lx.\n",
 963                           qedf->flags);
 964                 sc_cmd->result = DID_NO_CONNECT << 16;
 965                 sc_cmd->scsi_done(sc_cmd);
 966                 return 0;
 967         }
 968
 969         if (!qedf->pdev->msix_enabled) {
 970                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 971                     "Completing sc_cmd=%p DID_NO_CONNECT as MSI-X is not enabled.\n",
 972                     sc_cmd);
 973                 sc_cmd->result = DID_NO_CONNECT << 16;
 974                 sc_cmd->scsi_done(sc_cmd);
 975                 return 0;
 976         }
 977
 978         rval = fc_remote_port_chkready(rport);
 979         if (rval) {
 980                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 981                           "fc_remote_port_chkready failed=0x%x for port_id=0x%06x.\n",
 982                           rval, rport->port_id);
 983                 sc_cmd->result = rval;
 984                 sc_cmd->scsi_done(sc_cmd);
 985                 return 0;
 986         }
 987
 988         /* Retry command if we are doing a qed drain operation */
 989         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
 990                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Drain active.\n");
 991                 rc = SCSI_MLQUEUE_HOST_BUSY;
 992                 goto exit_qcmd;
 993         }
 994
 995         if (lport->state != LPORT_ST_READY ||
 996             atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
 997                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Link down.\n");
 998                 rc = SCSI_MLQUEUE_HOST_BUSY;
 999                 goto exit_qcmd;
1000         }
1001
1002         /* rport and tgt are allocated together, so tgt should be non-NULL */
1003         fcport = (struct qedf_rport *)&rp[1];
1004
1005         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
1006             test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
1007                 /*
1008                  * Session is not offloaded yet. Let SCSI-ml retry
1009                  * the command.
1010                  */
1011                 rc = SCSI_MLQUEUE_TARGET_BUSY;
1012                 goto exit_qcmd;
1013         }
1014
1015         atomic_inc(&fcport->ios_to_queue);
1016
1017         if (fcport->retry_delay_timestamp) {
1018                 /* Take fcport->rport_lock for resetting the delay_timestamp */
1019                 spin_lock_irqsave(&fcport->rport_lock, flags);
1020                 if (time_after(jiffies, fcport->retry_delay_timestamp)) {
1021                         fcport->retry_delay_timestamp = 0;
1022                 } else {
1023                         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1024                         /* If retry_delay timer is active, flow off the ML */
1025                         rc = SCSI_MLQUEUE_TARGET_BUSY;
1026                         atomic_dec(&fcport->ios_to_queue);
1027                         goto exit_qcmd;
1028                 }
1029                 spin_unlock_irqrestore(&fcport->rport_lock, flags);
1030         }
1031
1032         io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
1033         if (!io_req) {
1034                 rc = SCSI_MLQUEUE_HOST_BUSY;
1035                 atomic_dec(&fcport->ios_to_queue);
1036                 goto exit_qcmd;
1037         }
1038
1039         io_req->sc_cmd = sc_cmd;
1040
1041         /* Take fcport->rport_lock for posting to fcport send queue */
1042         spin_lock_irqsave(&fcport->rport_lock, flags);
1043         if (qedf_post_io_req(fcport, io_req)) {
1044                 QEDF_WARN(&(qedf->dbg_ctx), "Unable to post io_req\n");
1045                 /* Return SQE to pool */
1046                 atomic_inc(&fcport->free_sqes);
1047                 rc = SCSI_MLQUEUE_HOST_BUSY;
1048         }
1049         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1050         atomic_dec(&fcport->ios_to_queue);
1051
1052 exit_qcmd:
1053         return rc;
1054 }
1055
1056 static void qedf_parse_fcp_rsp(struct qedf_ioreq *io_req,
1057                                  struct fcoe_cqe_rsp_info *fcp_rsp)
1058 {
1059         struct scsi_cmnd *sc_cmd = io_req->sc_cmd;
1060         struct qedf_ctx *qedf = io_req->fcport->qedf;
1061         u8 rsp_flags = fcp_rsp->rsp_flags.flags;
1062         int fcp_sns_len = 0;
1063         int fcp_rsp_len = 0;
1064         uint8_t *rsp_info, *sense_data;
1065
1066         io_req->fcp_status = FC_GOOD;
1067         io_req->fcp_resid = 0;
1068         if (rsp_flags & (FCOE_FCP_RSP_FLAGS_FCP_RESID_OVER |
1069             FCOE_FCP_RSP_FLAGS_FCP_RESID_UNDER))
1070                 io_req->fcp_resid = fcp_rsp->fcp_resid;
1071
1072         io_req->scsi_comp_flags = rsp_flags;
1073         CMD_SCSI_STATUS(sc_cmd) = io_req->cdb_status =
1074             fcp_rsp->scsi_status_code;
1075
1076         if (rsp_flags &
1077             FCOE_FCP_RSP_FLAGS_FCP_RSP_LEN_VALID)
1078                 fcp_rsp_len = fcp_rsp->fcp_rsp_len;
1079
1080         if (rsp_flags &
1081             FCOE_FCP_RSP_FLAGS_FCP_SNS_LEN_VALID)
1082                 fcp_sns_len = fcp_rsp->fcp_sns_len;
1083
1084         io_req->fcp_rsp_len = fcp_rsp_len;
1085         io_req->fcp_sns_len = fcp_sns_len;
1086         rsp_info = sense_data = io_req->sense_buffer;
1087
1088         /* fetch fcp_rsp_code */
1089         if ((fcp_rsp_len == 4) || (fcp_rsp_len == 8)) {
1090                 /* Only for task management function */
1091                 io_req->fcp_rsp_code = rsp_info[3];
1092                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1093                     "fcp_rsp_code = %d\n", io_req->fcp_rsp_code);
1094                 /* Adjust sense-data location. */
1095                 sense_data += fcp_rsp_len;
1096         }
1097
1098         if (fcp_sns_len > SCSI_SENSE_BUFFERSIZE) {
1099                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1100                     "Truncating sense buffer\n");
1101                 fcp_sns_len = SCSI_SENSE_BUFFERSIZE;
1102         }
1103
1104         /* The sense buffer can be NULL for TMF commands */
1105         if (sc_cmd->sense_buffer) {
1106                 memset(sc_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
1107                 if (fcp_sns_len)
1108                         memcpy(sc_cmd->sense_buffer, sense_data,
1109                             fcp_sns_len);
1110         }
1111 }
1112
1113 static void qedf_unmap_sg_list(struct qedf_ctx *qedf, struct qedf_ioreq *io_req)
1114 {
1115         struct scsi_cmnd *sc = io_req->sc_cmd;
1116
1117         if (io_req->bd_tbl->bd_valid && sc && scsi_sg_count(sc)) {
1118                 dma_unmap_sg(&qedf->pdev->dev, scsi_sglist(sc),
1119                     scsi_sg_count(sc), sc->sc_data_direction);
1120                 io_req->bd_tbl->bd_valid = 0;
1121         }
1122 }
1123
1124 void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1125         struct qedf_ioreq *io_req)
1126 {
1127         struct scsi_cmnd *sc_cmd;
1128         struct fcoe_cqe_rsp_info *fcp_rsp;
1129         struct qedf_rport *fcport;
1130         int refcount;
1131         u16 scope, qualifier = 0;
1132         u8 fw_residual_flag = 0;
1133         unsigned long flags = 0;
1134         u16 chk_scope = 0;
1135
1136         if (!io_req)
1137                 return;
1138         if (!cqe)
1139                 return;
1140
1141         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1142             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
1143             test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
1144                 QEDF_ERR(&qedf->dbg_ctx,
1145                          "io_req xid=0x%x already in cleanup or abort processing or already completed.\n",
1146                          io_req->xid);
1147                 return;
1148         }
1149
1150         sc_cmd = io_req->sc_cmd;
1151         fcp_rsp = &cqe->cqe_info.rsp_info;
1152
1153         if (!sc_cmd) {
1154                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1155                 return;
1156         }
1157
1158         if (!sc_cmd->SCp.ptr) {
1159                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1160                     "another context.\n");
1161                 return;
1162         }
1163
1164         if (!sc_cmd->device) {
1165                 QEDF_ERR(&qedf->dbg_ctx,
1166                          "Device for sc_cmd %p is NULL.\n", sc_cmd);
1167                 return;
1168         }
1169
1170         if (!sc_cmd->request) {
1171                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd->request is NULL, "
1172                     "sc_cmd=%p.\n", sc_cmd);
1173                 return;
1174         }
1175
1176         if (!sc_cmd->request->q) {
1177                 QEDF_WARN(&(qedf->dbg_ctx), "request->q is NULL so request "
1178                    "is not valid, sc_cmd=%p.\n", sc_cmd);
1179                 return;
1180         }
1181
1182         fcport = io_req->fcport;
1183
1184         /*
1185          * When flush is active, let the cmds be completed from the cleanup
1186          * context
1187          */
1188         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
1189             (test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags) &&
1190              sc_cmd->device->lun == (u64)fcport->lun_reset_lun)) {
1191                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1192                           "Dropping good completion xid=0x%x as fcport is flushing",
1193                           io_req->xid);
1194                 return;
1195         }
1196
1197         qedf_parse_fcp_rsp(io_req, fcp_rsp);
1198
1199         qedf_unmap_sg_list(qedf, io_req);
1200
1201         /* Check for FCP transport error */
1202         if (io_req->fcp_rsp_len > 3 && io_req->fcp_rsp_code) {
1203                 QEDF_ERR(&(qedf->dbg_ctx),
1204                     "FCP I/O protocol failure xid=0x%x fcp_rsp_len=%d "
1205                     "fcp_rsp_code=%d.\n", io_req->xid, io_req->fcp_rsp_len,
1206                     io_req->fcp_rsp_code);
1207                 sc_cmd->result = DID_BUS_BUSY << 16;
1208                 goto out;
1209         }
1210
1211         fw_residual_flag = GET_FIELD(cqe->cqe_info.rsp_info.fw_error_flags,
1212             FCOE_CQE_RSP_INFO_FW_UNDERRUN);
1213         if (fw_residual_flag) {
1214                 QEDF_ERR(&qedf->dbg_ctx,
1215                          "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x fcp_resid=%d fw_residual=0x%x lba=%02x%02x%02x%02x.\n",
1216                          io_req->xid, fcp_rsp->rsp_flags.flags,
1217                          io_req->fcp_resid,
1218                          cqe->cqe_info.rsp_info.fw_residual, sc_cmd->cmnd[2],
1219                          sc_cmd->cmnd[3], sc_cmd->cmnd[4], sc_cmd->cmnd[5]);
1220
1221                 if (io_req->cdb_status == 0)
1222                         sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
1223                 else
1224                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1225
1226                 /*
1227                  * Set resid to the whole buffer length so we won't try to resue
1228                  * any previously data.
1229                  */
1230                 scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1231                 goto out;
1232         }
1233
1234         switch (io_req->fcp_status) {
1235         case FC_GOOD:
1236                 if (io_req->cdb_status == 0) {
1237                         /* Good I/O completion */
1238                         sc_cmd->result = DID_OK << 16;
1239                 } else {
1240                         refcount = kref_read(&io_req->refcount);
1241                         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1242                             "%d:0:%d:%lld xid=0x%0x op=0x%02x "
1243                             "lba=%02x%02x%02x%02x cdb_status=%d "
1244                             "fcp_resid=0x%x refcount=%d.\n",
1245                             qedf->lport->host->host_no, sc_cmd->device->id,
1246                             sc_cmd->device->lun, io_req->xid,
1247                             sc_cmd->cmnd[0], sc_cmd->cmnd[2], sc_cmd->cmnd[3],
1248                             sc_cmd->cmnd[4], sc_cmd->cmnd[5],
1249                             io_req->cdb_status, io_req->fcp_resid,
1250                             refcount);
1251                         sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
1252
1253                         if (io_req->cdb_status == SAM_STAT_TASK_SET_FULL ||
1254                             io_req->cdb_status == SAM_STAT_BUSY) {
1255                                 /*
1256                                  * Check whether we need to set retry_delay at
1257                                  * all based on retry_delay module parameter
1258                                  * and the status qualifier.
1259                                  */
1260
1261                                 /* Upper 2 bits */
1262                                 scope = fcp_rsp->retry_delay_timer & 0xC000;
1263                                 /* Lower 14 bits */
1264                                 qualifier = fcp_rsp->retry_delay_timer & 0x3FFF;
1265
1266                                 if (qedf_retry_delay)
1267                                         chk_scope = 1;
1268                                 /* Record stats */
1269                                 if (io_req->cdb_status ==
1270                                     SAM_STAT_TASK_SET_FULL)
1271                                         qedf->task_set_fulls++;
1272                                 else
1273                                         qedf->busy++;
1274                         }
1275                 }
1276                 if (io_req->fcp_resid)
1277                         scsi_set_resid(sc_cmd, io_req->fcp_resid);
1278
1279                 if (chk_scope == 1) {
1280                         if ((scope == 1 || scope == 2) &&
1281                             (qualifier > 0 && qualifier <= 0x3FEF)) {
1282                                 /* Check we don't go over the max */
1283                                 if (qualifier > QEDF_RETRY_DELAY_MAX) {
1284                                         qualifier = QEDF_RETRY_DELAY_MAX;
1285                                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1286                                                   "qualifier = %d\n",
1287                                                   (fcp_rsp->retry_delay_timer &
1288                                                   0x3FFF));
1289                                 }
1290                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1291                                           "Scope = %d and qualifier = %d",
1292                                           scope, qualifier);
1293                                 /*  Take fcport->rport_lock to
1294                                  *  update the retry_delay_timestamp
1295                                  */
1296                                 spin_lock_irqsave(&fcport->rport_lock, flags);
1297                                 fcport->retry_delay_timestamp =
1298                                         jiffies + (qualifier * HZ / 10);
1299                                 spin_unlock_irqrestore(&fcport->rport_lock,
1300                                                        flags);
1301
1302                         } else {
1303                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1304                                           "combination of scope = %d and qualifier = %d is not handled in qedf.\n",
1305                                           scope, qualifier);
1306                         }
1307                 }
1308                 break;
1309         default:
1310                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "fcp_status=%d.\n",
1311                            io_req->fcp_status);
1312                 break;
1313         }
1314
1315 out:
1316         if (qedf_io_tracing)
1317                 qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_RSP);
1318
1319         /*
1320          * We wait till the end of the function to clear the
1321          * outstanding bit in case we need to send an abort
1322          */
1323         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
1324
1325         io_req->sc_cmd = NULL;
1326         sc_cmd->SCp.ptr =  NULL;
1327         sc_cmd->scsi_done(sc_cmd);
1328         kref_put(&io_req->refcount, qedf_release_cmd);
1329 }
1330
1331 /* Return a SCSI command in some other context besides a normal completion */
1332 void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
1333         int result)
1334 {
1335         struct scsi_cmnd *sc_cmd;
1336         int refcount;
1337
1338         if (!io_req) {
1339                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "io_req is NULL\n");
1340                 return;
1341         }
1342
1343         if (test_and_set_bit(QEDF_CMD_ERR_SCSI_DONE, &io_req->flags)) {
1344                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1345                           "io_req:%p scsi_done handling already done\n",
1346                           io_req);
1347                 return;
1348         }
1349
1350         /*
1351          * We will be done with this command after this call so clear the
1352          * outstanding bit.
1353          */
1354         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
1355
1356         sc_cmd = io_req->sc_cmd;
1357
1358         if (!sc_cmd) {
1359                 QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd is NULL!\n");
1360                 return;
1361         }
1362
1363         if (!virt_addr_valid(sc_cmd)) {
1364                 QEDF_ERR(&qedf->dbg_ctx, "sc_cmd=%p is not valid.", sc_cmd);
1365                 goto bad_scsi_ptr;
1366         }
1367
1368         if (!sc_cmd->SCp.ptr) {
1369                 QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
1370                     "another context.\n");
1371                 return;
1372         }
1373
1374         if (!sc_cmd->device) {
1375                 QEDF_ERR(&qedf->dbg_ctx, "Device for sc_cmd %p is NULL.\n",
1376                          sc_cmd);
1377                 goto bad_scsi_ptr;
1378         }
1379
1380         if (!virt_addr_valid(sc_cmd->device)) {
1381                 QEDF_ERR(&qedf->dbg_ctx,
1382                          "Device pointer for sc_cmd %p is bad.\n", sc_cmd);
1383                 goto bad_scsi_ptr;
1384         }
1385
1386         if (!sc_cmd->sense_buffer) {
1387                 QEDF_ERR(&qedf->dbg_ctx,
1388                          "sc_cmd->sense_buffer for sc_cmd %p is NULL.\n",
1389                          sc_cmd);
1390                 goto bad_scsi_ptr;
1391         }
1392
1393         if (!virt_addr_valid(sc_cmd->sense_buffer)) {
1394                 QEDF_ERR(&qedf->dbg_ctx,
1395                          "sc_cmd->sense_buffer for sc_cmd %p is bad.\n",
1396                          sc_cmd);
1397                 goto bad_scsi_ptr;
1398         }
1399
1400         if (!sc_cmd->scsi_done) {
1401                 QEDF_ERR(&qedf->dbg_ctx,
1402                          "sc_cmd->scsi_done for sc_cmd %p is NULL.\n",
1403                          sc_cmd);
1404                 goto bad_scsi_ptr;
1405         }
1406
1407         qedf_unmap_sg_list(qedf, io_req);
1408
1409         sc_cmd->result = result << 16;
1410         refcount = kref_read(&io_req->refcount);
1411         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "%d:0:%d:%lld: Completing "
1412             "sc_cmd=%p result=0x%08x op=0x%02x lba=0x%02x%02x%02x%02x, "
1413             "allowed=%d retries=%d refcount=%d.\n",
1414             qedf->lport->host->host_no, sc_cmd->device->id,
1415             sc_cmd->device->lun, sc_cmd, sc_cmd->result, sc_cmd->cmnd[0],
1416             sc_cmd->cmnd[2], sc_cmd->cmnd[3], sc_cmd->cmnd[4],
1417             sc_cmd->cmnd[5], sc_cmd->allowed, sc_cmd->retries,
1418             refcount);
1419
1420         /*
1421          * Set resid to the whole buffer length so we won't try to resue any
1422          * previously read data
1423          */
1424         scsi_set_resid(sc_cmd, scsi_bufflen(sc_cmd));
1425
1426         if (qedf_io_tracing)
1427                 qedf_trace_io(io_req->fcport, io_req, QEDF_IO_TRACE_RSP);
1428
1429         io_req->sc_cmd = NULL;
1430         sc_cmd->SCp.ptr = NULL;
1431         sc_cmd->scsi_done(sc_cmd);
1432         kref_put(&io_req->refcount, qedf_release_cmd);
1433         return;
1434
1435 bad_scsi_ptr:
1436         /*
1437          * Clear the io_req->sc_cmd backpointer so we don't try to process
1438          * this again
1439          */
1440         io_req->sc_cmd = NULL;
1441         kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 001 */
1442 }
1443
1444 /*
1445  * Handle warning type CQE completions. This is mainly used for REC timer
1446  * popping.
1447  */
1448 void qedf_process_warning_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1449         struct qedf_ioreq *io_req)
1450 {
1451         int rval, i;
1452         struct qedf_rport *fcport = io_req->fcport;
1453         u64 err_warn_bit_map;
1454         u8 err_warn = 0xff;
1455
1456         if (!cqe) {
1457                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1458                           "cqe is NULL for io_req %p xid=0x%x\n",
1459                           io_req, io_req->xid);
1460                 return;
1461         }
1462
1463         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Warning CQE, "
1464                   "xid=0x%x\n", io_req->xid);
1465         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1466                   "err_warn_bitmap=%08x:%08x\n",
1467                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1468                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1469         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1470                   "rx_buff_off=%08x, rx_id=%04x\n",
1471                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1472                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1473                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1474
1475         /* Normalize the error bitmap value to an just an unsigned int */
1476         err_warn_bit_map = (u64)
1477             ((u64)cqe->cqe_info.err_info.err_warn_bitmap_hi << 32) |
1478             (u64)cqe->cqe_info.err_info.err_warn_bitmap_lo;
1479         for (i = 0; i < 64; i++) {
1480                 if (err_warn_bit_map & (u64)((u64)1 << i)) {
1481                         err_warn = i;
1482                         break;
1483                 }
1484         }
1485
1486         /* Check if REC TOV expired if this is a tape device */
1487         if (fcport->dev_type == QEDF_RPORT_TYPE_TAPE) {
1488                 if (err_warn ==
1489                     FCOE_WARNING_CODE_REC_TOV_TIMER_EXPIRATION) {
1490                         QEDF_ERR(&(qedf->dbg_ctx), "REC timer expired.\n");
1491                         if (!test_bit(QEDF_CMD_SRR_SENT, &io_req->flags)) {
1492                                 io_req->rx_buf_off =
1493                                     cqe->cqe_info.err_info.rx_buf_off;
1494                                 io_req->tx_buf_off =
1495                                     cqe->cqe_info.err_info.tx_buf_off;
1496                                 io_req->rx_id = cqe->cqe_info.err_info.rx_id;
1497                                 rval = qedf_send_rec(io_req);
1498                                 /*
1499                                  * We only want to abort the io_req if we
1500                                  * can't queue the REC command as we want to
1501                                  * keep the exchange open for recovery.
1502                                  */
1503                                 if (rval)
1504                                         goto send_abort;
1505                         }
1506                         return;
1507                 }
1508         }
1509
1510 send_abort:
1511         init_completion(&io_req->abts_done);
1512         rval = qedf_initiate_abts(io_req, true);
1513         if (rval)
1514                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1515 }
1516
1517 /* Cleanup a command when we receive an error detection completion */
1518 void qedf_process_error_detect(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1519         struct qedf_ioreq *io_req)
1520 {
1521         int rval;
1522
1523         if (!cqe) {
1524                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1525                           "cqe is NULL for io_req %p\n", io_req);
1526                 return;
1527         }
1528
1529         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "Error detection CQE, "
1530                   "xid=0x%x\n", io_req->xid);
1531         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx),
1532                   "err_warn_bitmap=%08x:%08x\n",
1533                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_hi),
1534                   le32_to_cpu(cqe->cqe_info.err_info.err_warn_bitmap_lo));
1535         QEDF_ERR(&(io_req->fcport->qedf->dbg_ctx), "tx_buff_off=%08x, "
1536                   "rx_buff_off=%08x, rx_id=%04x\n",
1537                   le32_to_cpu(cqe->cqe_info.err_info.tx_buf_off),
1538                   le32_to_cpu(cqe->cqe_info.err_info.rx_buf_off),
1539                   le32_to_cpu(cqe->cqe_info.err_info.rx_id));
1540
1541         if (qedf->stop_io_on_error) {
1542                 qedf_stop_all_io(qedf);
1543                 return;
1544         }
1545
1546         init_completion(&io_req->abts_done);
1547         rval = qedf_initiate_abts(io_req, true);
1548         if (rval)
1549                 QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
1550 }
1551
1552 static void qedf_flush_els_req(struct qedf_ctx *qedf,
1553         struct qedf_ioreq *els_req)
1554 {
1555         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1556             "Flushing ELS request xid=0x%x refcount=%d.\n", els_req->xid,
1557             kref_read(&els_req->refcount));
1558
1559         /*
1560          * Need to distinguish this from a timeout when calling the
1561          * els_req->cb_func.
1562          */
1563         els_req->event = QEDF_IOREQ_EV_ELS_FLUSH;
1564
1565         /* Cancel the timer */
1566         cancel_delayed_work_sync(&els_req->timeout_work);
1567
1568         /* Call callback function to complete command */
1569         if (els_req->cb_func && els_req->cb_arg) {
1570                 els_req->cb_func(els_req->cb_arg);
1571                 els_req->cb_arg = NULL;
1572         }
1573
1574         /* Release kref for original initiate_els */
1575         kref_put(&els_req->refcount, qedf_release_cmd);
1576 }
1577
1578 /* A value of -1 for lun is a wild card that means flush all
1579  * active SCSI I/Os for the target.
1580  */
1581 void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
1582 {
1583         struct qedf_ioreq *io_req;
1584         struct qedf_ctx *qedf;
1585         struct qedf_cmd_mgr *cmd_mgr;
1586         int i, rc;
1587         unsigned long flags;
1588         int flush_cnt = 0;
1589         int wait_cnt = 100;
1590         int refcount = 0;
1591
1592         if (!fcport) {
1593                 QEDF_ERR(NULL, "fcport is NULL\n");
1594                 return;
1595         }
1596
1597         /* Check that fcport is still offloaded */
1598         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1599                 QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
1600                 return;
1601         }
1602
1603         qedf = fcport->qedf;
1604
1605         if (!qedf) {
1606                 QEDF_ERR(NULL, "qedf is NULL.\n");
1607                 return;
1608         }
1609
1610         /* Only wait for all commands to be queued in the Upload context */
1611         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
1612             (lun == -1)) {
1613                 while (atomic_read(&fcport->ios_to_queue)) {
1614                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1615                                   "Waiting for %d I/Os to be queued\n",
1616                                   atomic_read(&fcport->ios_to_queue));
1617                         if (wait_cnt == 0) {
1618                                 QEDF_ERR(NULL,
1619                                          "%d IOs request could not be queued\n",
1620                                          atomic_read(&fcport->ios_to_queue));
1621                         }
1622                         msleep(20);
1623                         wait_cnt--;
1624                 }
1625         }
1626
1627         cmd_mgr = qedf->cmd_mgr;
1628
1629         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1630                   "Flush active i/o's num=0x%x fcport=0x%p port_id=0x%06x scsi_id=%d.\n",
1631                   atomic_read(&fcport->num_active_ios), fcport,
1632                   fcport->rdata->ids.port_id, fcport->rport->scsi_target_id);
1633         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Locking flush mutex.\n");
1634
1635         mutex_lock(&qedf->flush_mutex);
1636         if (lun == -1) {
1637                 set_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
1638         } else {
1639                 set_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
1640                 fcport->lun_reset_lun = lun;
1641         }
1642
1643         for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1644                 io_req = &cmd_mgr->cmds[i];
1645
1646                 if (!io_req)
1647                         continue;
1648                 if (!io_req->fcport)
1649                         continue;
1650
1651                 spin_lock_irqsave(&cmd_mgr->lock, flags);
1652
1653                 if (io_req->alloc) {
1654                         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
1655                                 if (io_req->cmd_type == QEDF_SCSI_CMD)
1656                                         QEDF_ERR(&qedf->dbg_ctx,
1657                                                  "Allocated but not queued, xid=0x%x\n",
1658                                                  io_req->xid);
1659                         }
1660                         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
1661                 } else {
1662                         spin_unlock_irqrestore(&cmd_mgr->lock, flags);
1663                         continue;
1664                 }
1665
1666                 if (io_req->fcport != fcport)
1667                         continue;
1668
1669                 /* In case of ABTS, CMD_OUTSTANDING is cleared on ABTS response,
1670                  * but RRQ is still pending.
1671                  * Workaround: Within qedf_send_rrq, we check if the fcport is
1672                  * NULL, and we drop the ref on the io_req to clean it up.
1673                  */
1674                 if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
1675                         refcount = kref_read(&io_req->refcount);
1676                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1677                                   "Not outstanding, xid=0x%x, cmd_type=%d refcount=%d.\n",
1678                                   io_req->xid, io_req->cmd_type, refcount);
1679                         /* If RRQ work has been queue, try to cancel it and
1680                          * free the io_req
1681                          */
1682                         if (atomic_read(&io_req->state) ==
1683                             QEDFC_CMD_ST_RRQ_WAIT) {
1684                                 if (cancel_delayed_work_sync
1685                                     (&io_req->rrq_work)) {
1686                                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1687                                                   "Putting reference for pending RRQ work xid=0x%x.\n",
1688                                                   io_req->xid);
1689                                         /* ID: 003 */
1690                                         kref_put(&io_req->refcount,
1691                                                  qedf_release_cmd);
1692                                 }
1693                         }
1694                         continue;
1695                 }
1696
1697                 /* Only consider flushing ELS during target reset */
1698                 if (io_req->cmd_type == QEDF_ELS &&
1699                     lun == -1) {
1700                         rc = kref_get_unless_zero(&io_req->refcount);
1701                         if (!rc) {
1702                                 QEDF_ERR(&(qedf->dbg_ctx),
1703                                     "Could not get kref for ELS io_req=0x%p xid=0x%x.\n",
1704                                     io_req, io_req->xid);
1705                                 continue;
1706                         }
1707                         flush_cnt++;
1708                         qedf_flush_els_req(qedf, io_req);
1709                         /*
1710                          * Release the kref and go back to the top of the
1711                          * loop.
1712                          */
1713                         goto free_cmd;
1714                 }
1715
1716                 if (io_req->cmd_type == QEDF_ABTS) {
1717                         /* ID: 004 */
1718                         rc = kref_get_unless_zero(&io_req->refcount);
1719                         if (!rc) {
1720                                 QEDF_ERR(&(qedf->dbg_ctx),
1721                                     "Could not get kref for abort io_req=0x%p xid=0x%x.\n",
1722                                     io_req, io_req->xid);
1723                                 continue;
1724                         }
1725                         if (lun != -1 && io_req->lun != lun)
1726                                 goto free_cmd;
1727
1728                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1729                             "Flushing abort xid=0x%x.\n", io_req->xid);
1730
1731                         if (cancel_delayed_work_sync(&io_req->rrq_work)) {
1732                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1733                                           "Putting ref for cancelled RRQ work xid=0x%x.\n",
1734                                           io_req->xid);
1735                                 kref_put(&io_req->refcount, qedf_release_cmd);
1736                         }
1737
1738                         if (cancel_delayed_work_sync(&io_req->timeout_work)) {
1739                                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1740                                           "Putting ref for cancelled tmo work xid=0x%x.\n",
1741                                           io_req->xid);
1742                                 qedf_initiate_cleanup(io_req, true);
1743                                 /* Notify eh_abort handler that ABTS is
1744                                  * complete
1745                                  */
1746                                 complete(&io_req->abts_done);
1747                                 clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1748                                 /* ID: 002 */
1749                                 kref_put(&io_req->refcount, qedf_release_cmd);
1750                         }
1751                         flush_cnt++;
1752                         goto free_cmd;
1753                 }
1754
1755                 if (!io_req->sc_cmd)
1756                         continue;
1757                 if (!io_req->sc_cmd->device) {
1758                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1759                                   "Device backpointer NULL for sc_cmd=%p.\n",
1760                                   io_req->sc_cmd);
1761                         /* Put reference for non-existent scsi_cmnd */
1762                         io_req->sc_cmd = NULL;
1763                         qedf_initiate_cleanup(io_req, false);
1764                         kref_put(&io_req->refcount, qedf_release_cmd);
1765                         continue;
1766                 }
1767                 if (lun > -1) {
1768                         if (io_req->lun != lun)
1769                                 continue;
1770                 }
1771
1772                 /*
1773                  * Use kref_get_unless_zero in the unlikely case the command
1774                  * we're about to flush was completed in the normal SCSI path
1775                  */
1776                 rc = kref_get_unless_zero(&io_req->refcount);
1777                 if (!rc) {
1778                         QEDF_ERR(&(qedf->dbg_ctx), "Could not get kref for "
1779                             "io_req=0x%p xid=0x%x\n", io_req, io_req->xid);
1780                         continue;
1781                 }
1782
1783                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
1784                     "Cleanup xid=0x%x.\n", io_req->xid);
1785                 flush_cnt++;
1786
1787                 /* Cleanup task and return I/O mid-layer */
1788                 qedf_initiate_cleanup(io_req, true);
1789
1790 free_cmd:
1791                 kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 004 */
1792         }
1793
1794         wait_cnt = 60;
1795         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1796                   "Flushed 0x%x I/Os, active=0x%x.\n",
1797                   flush_cnt, atomic_read(&fcport->num_active_ios));
1798         /* Only wait for all commands to complete in the Upload context */
1799         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
1800             (lun == -1)) {
1801                 while (atomic_read(&fcport->num_active_ios)) {
1802                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1803                                   "Flushed 0x%x I/Os, active=0x%x cnt=%d.\n",
1804                                   flush_cnt,
1805                                   atomic_read(&fcport->num_active_ios),
1806                                   wait_cnt);
1807                         if (wait_cnt == 0) {
1808                                 QEDF_ERR(&qedf->dbg_ctx,
1809                                          "Flushed %d I/Os, active=%d.\n",
1810                                          flush_cnt,
1811                                          atomic_read(&fcport->num_active_ios));
1812                                 for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
1813                                         io_req = &cmd_mgr->cmds[i];
1814                                         if (io_req->fcport &&
1815                                             io_req->fcport == fcport) {
1816                                                 refcount =
1817                                                 kref_read(&io_req->refcount);
1818                                                 set_bit(QEDF_CMD_DIRTY,
1819                                                         &io_req->flags);
1820                                                 QEDF_ERR(&qedf->dbg_ctx,
1821                                                          "Outstanding io_req =%p xid=0x%x flags=0x%lx, sc_cmd=%p refcount=%d cmd_type=%d.\n",
1822                                                          io_req, io_req->xid,
1823                                                          io_req->flags,
1824                                                          io_req->sc_cmd,
1825                                                          refcount,
1826                                                          io_req->cmd_type);
1827                                         }
1828                                 }
1829                                 WARN_ON(1);
1830                                 break;
1831                         }
1832                         msleep(500);
1833                         wait_cnt--;
1834                 }
1835         }
1836
1837         clear_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
1838         clear_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
1839         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Unlocking flush mutex.\n");
1840         mutex_unlock(&qedf->flush_mutex);
1841 }
1842
1843 /*
1844  * Initiate a ABTS middle path command. Note that we don't have to initialize
1845  * the task context for an ABTS task.
1846  */
1847 int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
1848 {
1849         struct fc_lport *lport;
1850         struct qedf_rport *fcport = io_req->fcport;
1851         struct fc_rport_priv *rdata;
1852         struct qedf_ctx *qedf;
1853         u16 xid;
1854         int rc = 0;
1855         unsigned long flags;
1856         struct fcoe_wqe *sqe;
1857         u16 sqe_idx;
1858         int refcount = 0;
1859
1860         /* Sanity check qedf_rport before dereferencing any pointers */
1861         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
1862                 QEDF_ERR(NULL, "tgt not offloaded\n");
1863                 rc = 1;
1864                 goto out;
1865         }
1866
1867         qedf = fcport->qedf;
1868         rdata = fcport->rdata;
1869
1870         if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
1871                 QEDF_ERR(&qedf->dbg_ctx, "stale rport\n");
1872                 rc = 1;
1873                 goto out;
1874         }
1875
1876         lport = qedf->lport;
1877
1878         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
1879                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
1880                 rc = 1;
1881                 goto drop_rdata_kref;
1882         }
1883
1884         if (atomic_read(&qedf->link_down_tmo_valid) > 0) {
1885                 QEDF_ERR(&(qedf->dbg_ctx), "link_down_tmo active.\n");
1886                 rc = 1;
1887                 goto drop_rdata_kref;
1888         }
1889
1890         /* Ensure room on SQ */
1891         if (!atomic_read(&fcport->free_sqes)) {
1892                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
1893                 rc = 1;
1894                 goto drop_rdata_kref;
1895         }
1896
1897         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
1898                 QEDF_ERR(&qedf->dbg_ctx, "fcport is uploading.\n");
1899                 rc = 1;
1900                 goto drop_rdata_kref;
1901         }
1902
1903         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
1904             test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
1905             test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
1906                 QEDF_ERR(&qedf->dbg_ctx,
1907                          "io_req xid=0x%x sc_cmd=%p already in cleanup or abort processing or already completed.\n",
1908                          io_req->xid, io_req->sc_cmd);
1909                 rc = 1;
1910                 goto drop_rdata_kref;
1911         }
1912
1913         kref_get(&io_req->refcount);
1914
1915         xid = io_req->xid;
1916         qedf->control_requests++;
1917         qedf->packet_aborts++;
1918
1919         /* Set the command type to abort */
1920         io_req->cmd_type = QEDF_ABTS;
1921         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
1922
1923         set_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
1924         refcount = kref_read(&io_req->refcount);
1925         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
1926                   "ABTS io_req xid = 0x%x refcount=%d\n",
1927                   xid, refcount);
1928
1929         qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT);
1930
1931         spin_lock_irqsave(&fcport->rport_lock, flags);
1932
1933         sqe_idx = qedf_get_sqe_idx(fcport);
1934         sqe = &fcport->sq[sqe_idx];
1935         memset(sqe, 0, sizeof(struct fcoe_wqe));
1936         io_req->task_params->sqe = sqe;
1937
1938         init_initiator_abort_fcoe_task(io_req->task_params);
1939         qedf_ring_doorbell(fcport);
1940
1941         spin_unlock_irqrestore(&fcport->rport_lock, flags);
1942
1943 drop_rdata_kref:
1944         kref_put(&rdata->kref, fc_rport_destroy);
1945 out:
1946         return rc;
1947 }
1948
1949 void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
1950         struct qedf_ioreq *io_req)
1951 {
1952         uint32_t r_ctl;
1953         int rc;
1954         struct qedf_rport *fcport = io_req->fcport;
1955
1956         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
1957                    "0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
1958
1959         r_ctl = cqe->cqe_info.abts_info.r_ctl;
1960
1961         /* This was added at a point when we were scheduling abts_compl &
1962          * cleanup_compl on different CPUs and there was a possibility of
1963          * the io_req to be freed from the other context before we got here.
1964          */
1965         if (!fcport) {
1966                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1967                           "Dropping ABTS completion xid=0x%x as fcport is NULL",
1968                           io_req->xid);
1969                 return;
1970         }
1971
1972         /*
1973          * When flush is active, let the cmds be completed from the cleanup
1974          * context
1975          */
1976         if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
1977             test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags)) {
1978                 QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
1979                           "Dropping ABTS completion xid=0x%x as fcport is flushing",
1980                           io_req->xid);
1981                 return;
1982         }
1983
1984         if (!cancel_delayed_work(&io_req->timeout_work)) {
1985                 QEDF_ERR(&qedf->dbg_ctx,
1986                          "Wasn't able to cancel abts timeout work.\n");
1987         }
1988
1989         switch (r_ctl) {
1990         case FC_RCTL_BA_ACC:
1991                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
1992                     "ABTS response - ACC Send RRQ after R_A_TOV\n");
1993                 io_req->event = QEDF_IOREQ_EV_ABORT_SUCCESS;
1994                 rc = kref_get_unless_zero(&io_req->refcount);   /* ID: 003 */
1995                 if (!rc) {
1996                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
1997                                   "kref is already zero so ABTS was already completed or flushed xid=0x%x.\n",
1998                                   io_req->xid);
1999                         return;
2000                 }
2001                 /*
2002                  * Dont release this cmd yet. It will be relesed
2003                  * after we get RRQ response
2004                  */
2005                 queue_delayed_work(qedf->dpc_wq, &io_req->rrq_work,
2006                     msecs_to_jiffies(qedf->lport->r_a_tov));
2007                 atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_WAIT);
2008                 break;
2009         /* For error cases let the cleanup return the command */
2010         case FC_RCTL_BA_RJT:
2011                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
2012                    "ABTS response - RJT\n");
2013                 io_req->event = QEDF_IOREQ_EV_ABORT_FAILED;
2014                 break;
2015         default:
2016                 QEDF_ERR(&(qedf->dbg_ctx), "Unknown ABTS response\n");
2017                 break;
2018         }
2019
2020         clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
2021
2022         if (io_req->sc_cmd) {
2023                 if (!io_req->return_scsi_cmd_on_abts)
2024                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2025                                   "Not call scsi_done for xid=0x%x.\n",
2026                                   io_req->xid);
2027                 if (io_req->return_scsi_cmd_on_abts)
2028                         qedf_scsi_done(qedf, io_req, DID_ERROR);
2029         }
2030
2031         /* Notify eh_abort handler that ABTS is complete */
2032         complete(&io_req->abts_done);
2033
2034         kref_put(&io_req->refcount, qedf_release_cmd);
2035 }
2036
2037 int qedf_init_mp_req(struct qedf_ioreq *io_req)
2038 {
2039         struct qedf_mp_req *mp_req;
2040         struct scsi_sge *mp_req_bd;
2041         struct scsi_sge *mp_resp_bd;
2042         struct qedf_ctx *qedf = io_req->fcport->qedf;
2043         dma_addr_t addr;
2044         uint64_t sz;
2045
2046         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_MP_REQ, "Entered.\n");
2047
2048         mp_req = (struct qedf_mp_req *)&(io_req->mp_req);
2049         memset(mp_req, 0, sizeof(struct qedf_mp_req));
2050
2051         if (io_req->cmd_type != QEDF_ELS) {
2052                 mp_req->req_len = sizeof(struct fcp_cmnd);
2053                 io_req->data_xfer_len = mp_req->req_len;
2054         } else
2055                 mp_req->req_len = io_req->data_xfer_len;
2056
2057         mp_req->req_buf = dma_alloc_coherent(&qedf->pdev->dev, QEDF_PAGE_SIZE,
2058             &mp_req->req_buf_dma, GFP_KERNEL);
2059         if (!mp_req->req_buf) {
2060                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req buffer\n");
2061                 qedf_free_mp_resc(io_req);
2062                 return -ENOMEM;
2063         }
2064
2065         mp_req->resp_buf = dma_alloc_coherent(&qedf->pdev->dev,
2066             QEDF_PAGE_SIZE, &mp_req->resp_buf_dma, GFP_KERNEL);
2067         if (!mp_req->resp_buf) {
2068                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc TM resp "
2069                           "buffer\n");
2070                 qedf_free_mp_resc(io_req);
2071                 return -ENOMEM;
2072         }
2073
2074         /* Allocate and map mp_req_bd and mp_resp_bd */
2075         sz = sizeof(struct scsi_sge);
2076         mp_req->mp_req_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
2077             &mp_req->mp_req_bd_dma, GFP_KERNEL);
2078         if (!mp_req->mp_req_bd) {
2079                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP req bd\n");
2080                 qedf_free_mp_resc(io_req);
2081                 return -ENOMEM;
2082         }
2083
2084         mp_req->mp_resp_bd = dma_alloc_coherent(&qedf->pdev->dev, sz,
2085             &mp_req->mp_resp_bd_dma, GFP_KERNEL);
2086         if (!mp_req->mp_resp_bd) {
2087                 QEDF_ERR(&(qedf->dbg_ctx), "Unable to alloc MP resp bd\n");
2088                 qedf_free_mp_resc(io_req);
2089                 return -ENOMEM;
2090         }
2091
2092         /* Fill bd table */
2093         addr = mp_req->req_buf_dma;
2094         mp_req_bd = mp_req->mp_req_bd;
2095         mp_req_bd->sge_addr.lo = U64_LO(addr);
2096         mp_req_bd->sge_addr.hi = U64_HI(addr);
2097         mp_req_bd->sge_len = QEDF_PAGE_SIZE;
2098
2099         /*
2100          * MP buffer is either a task mgmt command or an ELS.
2101          * So the assumption is that it consumes a single bd
2102          * entry in the bd table
2103          */
2104         mp_resp_bd = mp_req->mp_resp_bd;
2105         addr = mp_req->resp_buf_dma;
2106         mp_resp_bd->sge_addr.lo = U64_LO(addr);
2107         mp_resp_bd->sge_addr.hi = U64_HI(addr);
2108         mp_resp_bd->sge_len = QEDF_PAGE_SIZE;
2109
2110         return 0;
2111 }
2112
2113 /*
2114  * Last ditch effort to clear the port if it's stuck. Used only after a
2115  * cleanup task times out.
2116  */
2117 static void qedf_drain_request(struct qedf_ctx *qedf)
2118 {
2119         if (test_bit(QEDF_DRAIN_ACTIVE, &qedf->flags)) {
2120                 QEDF_ERR(&(qedf->dbg_ctx), "MCP drain already active.\n");
2121                 return;
2122         }
2123
2124         /* Set bit to return all queuecommand requests as busy */
2125         set_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
2126
2127         /* Call qed drain request for function. Should be synchronous */
2128         qed_ops->common->drain(qedf->cdev);
2129
2130         /* Settle time for CQEs to be returned */
2131         msleep(100);
2132
2133         /* Unplug and continue */
2134         clear_bit(QEDF_DRAIN_ACTIVE, &qedf->flags);
2135 }
2136
2137 /*
2138  * Returns SUCCESS if the cleanup task does not timeout, otherwise return
2139  * FAILURE.
2140  */
2141 int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
2142         bool return_scsi_cmd_on_abts)
2143 {
2144         struct qedf_rport *fcport;
2145         struct qedf_ctx *qedf;
2146         int tmo = 0;
2147         int rc = SUCCESS;
2148         unsigned long flags;
2149         struct fcoe_wqe *sqe;
2150         u16 sqe_idx;
2151         int refcount = 0;
2152
2153         fcport = io_req->fcport;
2154         if (!fcport) {
2155                 QEDF_ERR(NULL, "fcport is NULL.\n");
2156                 return SUCCESS;
2157         }
2158
2159         /* Sanity check qedf_rport before dereferencing any pointers */
2160         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
2161                 QEDF_ERR(NULL, "tgt not offloaded\n");
2162                 rc = 1;
2163                 return SUCCESS;
2164         }
2165
2166         qedf = fcport->qedf;
2167         if (!qedf) {
2168                 QEDF_ERR(NULL, "qedf is NULL.\n");
2169                 return SUCCESS;
2170         }
2171
2172         if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
2173             test_and_set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
2174                 QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
2175                           "cleanup processing or already completed.\n",
2176                           io_req->xid);
2177                 return SUCCESS;
2178         }
2179         set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2180
2181         /* Ensure room on SQ */
2182         if (!atomic_read(&fcport->free_sqes)) {
2183                 QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
2184                 /* Need to make sure we clear the flag since it was set */
2185                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2186                 return FAILED;
2187         }
2188
2189         if (io_req->cmd_type == QEDF_CLEANUP) {
2190                 QEDF_ERR(&qedf->dbg_ctx,
2191                          "io_req=0x%x is already a cleanup command cmd_type=%d.\n",
2192                          io_req->xid, io_req->cmd_type);
2193                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2194                 return SUCCESS;
2195         }
2196
2197         refcount = kref_read(&io_req->refcount);
2198
2199         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
2200                   "Entered xid=0x%x sc_cmd=%p cmd_type=%d flags=0x%lx refcount=%d fcport=%p port_id=0x%06x\n",
2201                   io_req->xid, io_req->sc_cmd, io_req->cmd_type, io_req->flags,
2202                   refcount, fcport, fcport->rdata->ids.port_id);
2203
2204         /* Cleanup cmds re-use the same TID as the original I/O */
2205         io_req->cmd_type = QEDF_CLEANUP;
2206         io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
2207
2208         init_completion(&io_req->cleanup_done);
2209
2210         spin_lock_irqsave(&fcport->rport_lock, flags);
2211
2212         sqe_idx = qedf_get_sqe_idx(fcport);
2213         sqe = &fcport->sq[sqe_idx];
2214         memset(sqe, 0, sizeof(struct fcoe_wqe));
2215         io_req->task_params->sqe = sqe;
2216
2217         init_initiator_cleanup_fcoe_task(io_req->task_params);
2218         qedf_ring_doorbell(fcport);
2219
2220         spin_unlock_irqrestore(&fcport->rport_lock, flags);
2221
2222         tmo = wait_for_completion_timeout(&io_req->cleanup_done,
2223                                           QEDF_CLEANUP_TIMEOUT * HZ);
2224
2225         if (!tmo) {
2226                 rc = FAILED;
2227                 /* Timeout case */
2228                 QEDF_ERR(&(qedf->dbg_ctx), "Cleanup command timeout, "
2229                           "xid=%x.\n", io_req->xid);
2230                 clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2231                 /* Issue a drain request if cleanup task times out */
2232                 QEDF_ERR(&(qedf->dbg_ctx), "Issuing MCP drain request.\n");
2233                 qedf_drain_request(qedf);
2234         }
2235
2236         /* If it TASK MGMT handle it, reference will be decreased
2237          * in qedf_execute_tmf
2238          */
2239         if (io_req->tm_flags  == FCP_TMF_LUN_RESET ||
2240             io_req->tm_flags == FCP_TMF_TGT_RESET) {
2241                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2242                 io_req->sc_cmd = NULL;
2243                 complete(&io_req->tm_done);
2244         }
2245
2246         if (io_req->sc_cmd) {
2247                 if (!io_req->return_scsi_cmd_on_abts)
2248                         QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
2249                                   "Not call scsi_done for xid=0x%x.\n",
2250                                   io_req->xid);
2251                 if (io_req->return_scsi_cmd_on_abts)
2252                         qedf_scsi_done(qedf, io_req, DID_ERROR);
2253         }
2254
2255         if (rc == SUCCESS)
2256                 io_req->event = QEDF_IOREQ_EV_CLEANUP_SUCCESS;
2257         else
2258                 io_req->event = QEDF_IOREQ_EV_CLEANUP_FAILED;
2259
2260         return rc;
2261 }
2262
2263 void qedf_process_cleanup_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
2264         struct qedf_ioreq *io_req)
2265 {
2266         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid = 0x%x\n",
2267                    io_req->xid);
2268
2269         clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
2270
2271         /* Complete so we can finish cleaning up the I/O */
2272         complete(&io_req->cleanup_done);
2273 }
2274
2275 static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
2276         uint8_t tm_flags)
2277 {
2278         struct qedf_ioreq *io_req;
2279         struct e4_fcoe_task_context *task;
2280         struct qedf_ctx *qedf = fcport->qedf;
2281         struct fc_lport *lport = qedf->lport;
2282         int rc = 0;
2283         uint16_t xid;
2284         int tmo = 0;
2285         int lun = 0;
2286         unsigned long flags;
2287         struct fcoe_wqe *sqe;
2288         u16 sqe_idx;
2289
2290         if (!sc_cmd) {
2291                 QEDF_ERR(&qedf->dbg_ctx, "sc_cmd is NULL\n");
2292                 return FAILED;
2293         }
2294
2295         lun = (int)sc_cmd->device->lun;
2296         if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
2297                 QEDF_ERR(&(qedf->dbg_ctx), "fcport not offloaded\n");
2298                 rc = FAILED;
2299                 goto no_flush;
2300         }
2301
2302         io_req = qedf_alloc_cmd(fcport, QEDF_TASK_MGMT_CMD);
2303         if (!io_req) {
2304                 QEDF_ERR(&(qedf->dbg_ctx), "Failed TMF");
2305                 rc = -EAGAIN;
2306                 goto no_flush;
2307         }
2308
2309         if (tm_flags == FCP_TMF_LUN_RESET)
2310                 qedf->lun_resets++;
2311         else if (tm_flags == FCP_TMF_TGT_RESET)
2312                 qedf->target_resets++;
2313
2314         /* Initialize rest of io_req fields */
2315         io_req->sc_cmd = sc_cmd;
2316         io_req->fcport = fcport;
2317         io_req->cmd_type = QEDF_TASK_MGMT_CMD;
2318
2319         /* Record which cpu this request is associated with */
2320         io_req->cpu = smp_processor_id();
2321
2322         /* Set TM flags */
2323         io_req->io_req_flags = QEDF_READ;
2324         io_req->data_xfer_len = 0;
2325         io_req->tm_flags = tm_flags;
2326
2327         /* Default is to return a SCSI command when an error occurs */
2328         io_req->return_scsi_cmd_on_abts = false;
2329
2330         /* Obtain exchange id */
2331         xid = io_req->xid;
2332
2333         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "TMF io_req xid = "
2334                    "0x%x\n", xid);
2335
2336         /* Initialize task context for this IO request */
2337         task = qedf_get_task_mem(&qedf->tasks, xid);
2338
2339         init_completion(&io_req->tm_done);
2340
2341         spin_lock_irqsave(&fcport->rport_lock, flags);
2342
2343         sqe_idx = qedf_get_sqe_idx(fcport);
2344         sqe = &fcport->sq[sqe_idx];
2345         memset(sqe, 0, sizeof(struct fcoe_wqe));
2346
2347         qedf_init_task(fcport, lport, io_req, task, sqe);
2348         qedf_ring_doorbell(fcport);
2349
2350         spin_unlock_irqrestore(&fcport->rport_lock, flags);
2351
2352         set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2353         tmo = wait_for_completion_timeout(&io_req->tm_done,
2354             QEDF_TM_TIMEOUT * HZ);
2355
2356         if (!tmo) {
2357                 rc = FAILED;
2358                 QEDF_ERR(&(qedf->dbg_ctx), "wait for tm_cmpl timeout!\n");
2359                 /* Clear outstanding bit since command timed out */
2360                 clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2361                 io_req->sc_cmd = NULL;
2362         } else {
2363                 /* Check TMF response code */
2364                 if (io_req->fcp_rsp_code == 0)
2365                         rc = SUCCESS;
2366                 else
2367                         rc = FAILED;
2368         }
2369         /*
2370          * Double check that fcport has not gone into an uploading state before
2371          * executing the command flush for the LUN/target.
2372          */
2373         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2374                 QEDF_ERR(&qedf->dbg_ctx,
2375                          "fcport is uploading, not executing flush.\n");
2376                 goto no_flush;
2377         }
2378         /* We do not need this io_req any more */
2379         kref_put(&io_req->refcount, qedf_release_cmd);
2380
2381
2382         if (tm_flags == FCP_TMF_LUN_RESET)
2383                 qedf_flush_active_ios(fcport, lun);
2384         else
2385                 qedf_flush_active_ios(fcport, -1);
2386
2387 no_flush:
2388         if (rc != SUCCESS) {
2389                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command failed...\n");
2390                 rc = FAILED;
2391         } else {
2392                 QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command success...\n");
2393                 rc = SUCCESS;
2394         }
2395         return rc;
2396 }
2397
2398 int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
2399 {
2400         struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
2401         struct fc_rport_libfc_priv *rp = rport->dd_data;
2402         struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
2403         struct qedf_ctx *qedf;
2404         struct fc_lport *lport = shost_priv(sc_cmd->device->host);
2405         int rc = SUCCESS;
2406         int rval;
2407         struct qedf_ioreq *io_req = NULL;
2408         int ref_cnt = 0;
2409         struct fc_rport_priv *rdata = fcport->rdata;
2410
2411         QEDF_ERR(NULL,
2412                  "tm_flags 0x%x sc_cmd %p op = 0x%02x target_id = 0x%x lun=%d\n",
2413                  tm_flags, sc_cmd, sc_cmd->cmd_len ? sc_cmd->cmnd[0] : 0xff,
2414                  rport->scsi_target_id, (int)sc_cmd->device->lun);
2415
2416         if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
2417                 QEDF_ERR(NULL, "stale rport\n");
2418                 return FAILED;
2419         }
2420
2421         QEDF_ERR(NULL, "portid=%06x tm_flags =%s\n", rdata->ids.port_id,
2422                  (tm_flags == FCP_TMF_TGT_RESET) ? "TARGET RESET" :
2423                  "LUN RESET");
2424
2425         if (sc_cmd->SCp.ptr) {
2426                 io_req = (struct qedf_ioreq *)sc_cmd->SCp.ptr;
2427                 ref_cnt = kref_read(&io_req->refcount);
2428                 QEDF_ERR(NULL,
2429                          "orig io_req = %p xid = 0x%x ref_cnt = %d.\n",
2430                          io_req, io_req->xid, ref_cnt);
2431         }
2432
2433         rval = fc_remote_port_chkready(rport);
2434         if (rval) {
2435                 QEDF_ERR(NULL, "device_reset rport not ready\n");
2436                 rc = FAILED;
2437                 goto tmf_err;
2438         }
2439
2440         rc = fc_block_scsi_eh(sc_cmd);
2441         if (rc)
2442                 goto tmf_err;
2443
2444         if (!fcport) {
2445                 QEDF_ERR(NULL, "device_reset: rport is NULL\n");
2446                 rc = FAILED;
2447                 goto tmf_err;
2448         }
2449
2450         qedf = fcport->qedf;
2451
2452         if (!qedf) {
2453                 QEDF_ERR(NULL, "qedf is NULL.\n");
2454                 rc = FAILED;
2455                 goto tmf_err;
2456         }
2457
2458         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2459                 QEDF_ERR(&qedf->dbg_ctx, "Connection is getting uploaded.\n");
2460                 rc = SUCCESS;
2461                 goto tmf_err;
2462         }
2463
2464         if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
2465             test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
2466                 rc = SUCCESS;
2467                 goto tmf_err;
2468         }
2469
2470         if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
2471                 QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
2472                 rc = FAILED;
2473                 goto tmf_err;
2474         }
2475
2476         if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
2477                 if (!fcport->rdata)
2478                         QEDF_ERR(&qedf->dbg_ctx, "fcport %p is uploading.\n",
2479                                  fcport);
2480                 else
2481                         QEDF_ERR(&qedf->dbg_ctx,
2482                                  "fcport %p port_id=%06x is uploading.\n",
2483                                  fcport, fcport->rdata->ids.port_id);
2484                 rc = FAILED;
2485                 goto tmf_err;
2486         }
2487
2488         rc = qedf_execute_tmf(fcport, sc_cmd, tm_flags);
2489
2490 tmf_err:
2491         kref_put(&rdata->kref, fc_rport_destroy);
2492         return rc;
2493 }
2494
2495 void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
2496         struct qedf_ioreq *io_req)
2497 {
2498         struct fcoe_cqe_rsp_info *fcp_rsp;
2499
2500         clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
2501
2502         fcp_rsp = &cqe->cqe_info.rsp_info;
2503         qedf_parse_fcp_rsp(io_req, fcp_rsp);
2504
2505         io_req->sc_cmd = NULL;
2506         complete(&io_req->tm_done);
2507 }
2508
2509 void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
2510         struct fcoe_cqe *cqe)
2511 {
2512         unsigned long flags;
2513         uint16_t pktlen = cqe->cqe_info.unsolic_info.pkt_len;
2514         u32 payload_len, crc;
2515         struct fc_frame_header *fh;
2516         struct fc_frame *fp;
2517         struct qedf_io_work *io_work;
2518         u32 bdq_idx;
2519         void *bdq_addr;
2520         struct scsi_bd *p_bd_info;
2521
2522         p_bd_info = &cqe->cqe_info.unsolic_info.bd_info;
2523         QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2524                   "address.hi=%x, address.lo=%x, opaque_data.hi=%x, opaque_data.lo=%x, bdq_prod_idx=%u, len=%u\n",
2525                   le32_to_cpu(p_bd_info->address.hi),
2526                   le32_to_cpu(p_bd_info->address.lo),
2527                   le32_to_cpu(p_bd_info->opaque.fcoe_opaque.hi),
2528                   le32_to_cpu(p_bd_info->opaque.fcoe_opaque.lo),
2529                   qedf->bdq_prod_idx, pktlen);
2530
2531         bdq_idx = le32_to_cpu(p_bd_info->opaque.fcoe_opaque.lo);
2532         if (bdq_idx >= QEDF_BDQ_SIZE) {
2533                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_idx is out of range %d.\n",
2534                     bdq_idx);
2535                 goto increment_prod;
2536         }
2537
2538         bdq_addr = qedf->bdq[bdq_idx].buf_addr;
2539         if (!bdq_addr) {
2540                 QEDF_ERR(&(qedf->dbg_ctx), "bdq_addr is NULL, dropping "
2541                     "unsolicited packet.\n");
2542                 goto increment_prod;
2543         }
2544
2545         if (qedf_dump_frames) {
2546                 QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_UNSOL,
2547                     "BDQ frame is at addr=%p.\n", bdq_addr);
2548                 print_hex_dump(KERN_WARNING, "bdq ", DUMP_PREFIX_OFFSET, 16, 1,
2549                     (void *)bdq_addr, pktlen, false);
2550         }
2551
2552         /* Allocate frame */
2553         payload_len = pktlen - sizeof(struct fc_frame_header);
2554         fp = fc_frame_alloc(qedf->lport, payload_len);
2555         if (!fp) {
2556                 QEDF_ERR(&(qedf->dbg_ctx), "Could not allocate fp.\n");
2557                 goto increment_prod;
2558         }
2559
2560         /* Copy data from BDQ buffer into fc_frame struct */
2561         fh = (struct fc_frame_header *)fc_frame_header_get(fp);
2562         memcpy(fh, (void *)bdq_addr, pktlen);
2563
2564         QEDF_WARN(&qedf->dbg_ctx,
2565                   "Processing Unsolicated frame, src=%06x dest=%06x r_ctl=0x%x type=0x%x cmd=%02x\n",
2566                   ntoh24(fh->fh_s_id), ntoh24(fh->fh_d_id), fh->fh_r_ctl,
2567                   fh->fh_type, fc_frame_payload_op(fp));
2568
2569         /* Initialize the frame so libfc sees it as a valid frame */
2570         crc = fcoe_fc_crc(fp);
2571         fc_frame_init(fp);
2572         fr_dev(fp) = qedf->lport;
2573         fr_sof(fp) = FC_SOF_I3;
2574         fr_eof(fp) = FC_EOF_T;
2575         fr_crc(fp) = cpu_to_le32(~crc);
2576
2577         /*
2578          * We need to return the frame back up to libfc in a non-atomic
2579          * context
2580          */
2581         io_work = mempool_alloc(qedf->io_mempool, GFP_ATOMIC);
2582         if (!io_work) {
2583                 QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate "
2584                            "work for I/O completion.\n");
2585                 fc_frame_free(fp);
2586                 goto increment_prod;
2587         }
2588         memset(io_work, 0, sizeof(struct qedf_io_work));
2589
2590         INIT_WORK(&io_work->work, qedf_fp_io_handler);
2591
2592         /* Copy contents of CQE for deferred processing */
2593         memcpy(&io_work->cqe, cqe, sizeof(struct fcoe_cqe));
2594
2595         io_work->qedf = qedf;
2596         io_work->fp = fp;
2597
2598         queue_work_on(smp_processor_id(), qedf_io_wq, &io_work->work);
2599 increment_prod:
2600         spin_lock_irqsave(&qedf->hba_lock, flags);
2601
2602         /* Increment producer to let f/w know we've handled the frame */
2603         qedf->bdq_prod_idx++;
2604
2605         /* Producer index wraps at uint16_t boundary */
2606         if (qedf->bdq_prod_idx == 0xffff)
2607                 qedf->bdq_prod_idx = 0;
2608
2609         writew(qedf->bdq_prod_idx, qedf->bdq_primary_prod);
2610         readw(qedf->bdq_primary_prod);
2611         writew(qedf->bdq_prod_idx, qedf->bdq_secondary_prod);
2612         readw(qedf->bdq_secondary_prod);
2613
2614         spin_unlock_irqrestore(&qedf->hba_lock, flags);
2615 }