drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*******************************************************************************
   3  * IBM Virtual SCSI Target Driver
   4  * Copyright (C) 2003-2005 Dave Boutcher ([email protected]) IBM Corp.
   5  *                         Santiago Leon ([email protected]) IBM Corp.
   6  *                         Linda Xie ([email protected]) IBM Corp.
   7  *
   8  * Copyright (C) 2005-2011 FUJITA Tomonori <[email protected]>
   9  * Copyright (C) 2010 Nicholas A. Bellinger <[email protected]>
  10  *
  11  * Authors: Bryant G. Ly <[email protected]>
  12  * Authors: Michael Cyr <[email protected]>
  13  *
  14  ****************************************************************************/
  15
  16 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  17
  18 #include <linux/module.h>
  19 #include <linux/kernel.h>
  20 #include <linux/slab.h>
  21 #include <linux/types.h>
  22 #include <linux/list.h>
  23 #include <linux/string.h>
  24 #include <linux/delay.h>
  25
  26 #include <target/target_core_base.h>
  27 #include <target/target_core_fabric.h>
  28
  29 #include <asm/hvcall.h>
  30 #include <asm/vio.h>
  31
  32 #include <scsi/viosrp.h>
  33
  34 #include "ibmvscsi_tgt.h"
  35
  36 #define IBMVSCSIS_VERSION       "v0.2"
  37
  38 #define INITIAL_SRP_LIMIT       800
  39 #define DEFAULT_MAX_SECTORS     256
  40 #define MAX_TXU                 1024 * 1024
  41
  42 static uint max_vdma_size = MAX_H_COPY_RDMA;
  43
  44 static char system_id[SYS_ID_NAME_LEN] = "";
  45 static char partition_name[PARTITION_NAMELEN] = "UNKNOWN";
  46 static uint partition_number = -1;
  47
  48 /* Adapter list and lock to control it */
  49 static DEFINE_SPINLOCK(ibmvscsis_dev_lock);
  50 static LIST_HEAD(ibmvscsis_dev_list);
  51
  52 static long ibmvscsis_parse_command(struct scsi_info *vscsi,
  53                                     struct viosrp_crq *crq);
  54
  55 static void ibmvscsis_adapter_idle(struct scsi_info *vscsi);
  56
  57 static void ibmvscsis_determine_resid(struct se_cmd *se_cmd,
  58                                       struct srp_rsp *rsp)
  59 {
  60         u32 residual_count = se_cmd->residual_count;
  61
  62         if (!residual_count)
  63                 return;
  64
  65         if (se_cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
  66                 if (se_cmd->data_direction == DMA_TO_DEVICE) {
  67                         /* residual data from an underflow write */
  68                         rsp->flags = SRP_RSP_FLAG_DOUNDER;
  69                         rsp->data_out_res_cnt = cpu_to_be32(residual_count);
  70                 } else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
  71                         /* residual data from an underflow read */
  72                         rsp->flags = SRP_RSP_FLAG_DIUNDER;
  73                         rsp->data_in_res_cnt = cpu_to_be32(residual_count);
  74                 }
  75         } else if (se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
  76                 if (se_cmd->data_direction == DMA_TO_DEVICE) {
  77                         /* residual data from an overflow write */
  78                         rsp->flags = SRP_RSP_FLAG_DOOVER;
  79                         rsp->data_out_res_cnt = cpu_to_be32(residual_count);
  80                 } else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
  81                         /* residual data from an overflow read */
  82                         rsp->flags = SRP_RSP_FLAG_DIOVER;
  83                         rsp->data_in_res_cnt = cpu_to_be32(residual_count);
  84                 }
  85         }
  86 }
  87
  88 /**
  89  * connection_broken() - Determine if the connection to the client is good
  90  * @vscsi:      Pointer to our adapter structure
  91  *
  92  * This function attempts to send a ping MAD to the client. If the call to
  93  * queue the request returns H_CLOSED then the connection has been broken
  94  * and the function returns TRUE.
  95  *
  96  * EXECUTION ENVIRONMENT:
  97  *      Interrupt or Process environment
  98  */
  99 static bool connection_broken(struct scsi_info *vscsi)
 100 {
 101         struct viosrp_crq *crq;
 102         u64 buffer[2] = { 0, 0 };
 103         long h_return_code;
 104         bool rc = false;
 105
 106         /* create a PING crq */
 107         crq = (struct viosrp_crq *)&buffer;
 108         crq->valid = VALID_CMD_RESP_EL;
 109         crq->format = MESSAGE_IN_CRQ;
 110         crq->status = PING;
 111
 112         h_return_code = h_send_crq(vscsi->dds.unit_id,
 113                                    cpu_to_be64(buffer[MSG_HI]),
 114                                    cpu_to_be64(buffer[MSG_LOW]));
 115
 116         dev_dbg(&vscsi->dev, "Connection_broken: rc %ld\n", h_return_code);
 117
 118         if (h_return_code == H_CLOSED)
 119                 rc = true;
 120
 121         return rc;
 122 }
 123
 124 /**
 125  * ibmvscsis_unregister_command_q() - Helper Function-Unregister Command Queue
 126  * @vscsi:      Pointer to our adapter structure
 127  *
 128  * This function calls h_free_q then frees the interrupt bit etc.
 129  * It must release the lock before doing so because of the time it can take
 130  * for h_free_crq in PHYP
 131  * NOTE: the caller must make sure that state and or flags will prevent
 132  *       interrupt handler from scheduling work.
 133  * NOTE: anyone calling this function may need to set the CRQ_CLOSED flag
 134  *       we can't do it here, because we don't have the lock
 135  *
 136  * EXECUTION ENVIRONMENT:
 137  *      Process level
 138  */
 139 static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi)
 140 {
 141         long qrc;
 142         long rc = ADAPT_SUCCESS;
 143         int ticks = 0;
 144
 145         do {
 146                 qrc = h_free_crq(vscsi->dds.unit_id);
 147                 switch (qrc) {
 148                 case H_SUCCESS:
 149                         spin_lock_bh(&vscsi->intr_lock);
 150                         vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS;
 151                         spin_unlock_bh(&vscsi->intr_lock);
 152                         break;
 153
 154                 case H_HARDWARE:
 155                 case H_PARAMETER:
 156                         dev_err(&vscsi->dev, "unregister_command_q: error from h_free_crq %ld\n",
 157                                 qrc);
 158                         rc = ERROR;
 159                         break;
 160
 161                 case H_BUSY:
 162                 case H_LONG_BUSY_ORDER_1_MSEC:
 163                         /* msleep not good for small values */
 164                         usleep_range(1000, 2000);
 165                         ticks += 1;
 166                         break;
 167                 case H_LONG_BUSY_ORDER_10_MSEC:
 168                         usleep_range(10000, 20000);
 169                         ticks += 10;
 170                         break;
 171                 case H_LONG_BUSY_ORDER_100_MSEC:
 172                         msleep(100);
 173                         ticks += 100;
 174                         break;
 175                 case H_LONG_BUSY_ORDER_1_SEC:
 176                         ssleep(1);
 177                         ticks += 1000;
 178                         break;
 179                 case H_LONG_BUSY_ORDER_10_SEC:
 180                         ssleep(10);
 181                         ticks += 10000;
 182                         break;
 183                 case H_LONG_BUSY_ORDER_100_SEC:
 184                         ssleep(100);
 185                         ticks += 100000;
 186                         break;
 187                 default:
 188                         dev_err(&vscsi->dev, "unregister_command_q: unknown error %ld from h_free_crq\n",
 189                                 qrc);
 190                         rc = ERROR;
 191                         break;
 192                 }
 193
 194                 /*
 195                  * dont wait more then 300 seconds
 196                  * ticks are in milliseconds more or less
 197                  */
 198                 if (ticks > 300000 && qrc != H_SUCCESS) {
 199                         rc = ERROR;
 200                         dev_err(&vscsi->dev, "Excessive wait for h_free_crq\n");
 201                 }
 202         } while (qrc != H_SUCCESS && rc == ADAPT_SUCCESS);
 203
 204         dev_dbg(&vscsi->dev, "Freeing CRQ: phyp rc %ld, rc %ld\n", qrc, rc);
 205
 206         return rc;
 207 }
 208
 209 /**
 210  * ibmvscsis_delete_client_info() - Helper function to Delete Client Info
 211  * @vscsi:      Pointer to our adapter structure
 212  * @client_closed:      True if client closed its queue
 213  *
 214  * Deletes information specific to the client when the client goes away
 215  *
 216  * EXECUTION ENVIRONMENT:
 217  *      Interrupt or Process
 218  */
 219 static void ibmvscsis_delete_client_info(struct scsi_info *vscsi,
 220                                          bool client_closed)
 221 {
 222         vscsi->client_cap = 0;
 223
 224         /*
 225          * Some things we don't want to clear if we're closing the queue,
 226          * because some clients don't resend the host handshake when they
 227          * get a transport event.
 228          */
 229         if (client_closed)
 230                 vscsi->client_data.os_type = 0;
 231 }
 232
 233 /**
 234  * ibmvscsis_free_command_q() - Free Command Queue
 235  * @vscsi:      Pointer to our adapter structure
 236  *
 237  * This function calls unregister_command_q, then clears interrupts and
 238  * any pending interrupt acknowledgments associated with the command q.
 239  * It also clears memory if there is no error.
 240  *
 241  * PHYP did not meet the PAPR architecture so that we must give up the
 242  * lock. This causes a timing hole regarding state change.  To close the
 243  * hole this routine does accounting on any change that occurred during
 244  * the time the lock is not held.
 245  * NOTE: must give up and then acquire the interrupt lock, the caller must
 246  *       make sure that state and or flags will prevent interrupt handler from
 247  *       scheduling work.
 248  *
 249  * EXECUTION ENVIRONMENT:
 250  *      Process level, interrupt lock is held
 251  */
 252 static long ibmvscsis_free_command_q(struct scsi_info *vscsi)
 253 {
 254         int bytes;
 255         u32 flags_under_lock;
 256         u16 state_under_lock;
 257         long rc = ADAPT_SUCCESS;
 258
 259         if (!(vscsi->flags & CRQ_CLOSED)) {
 260                 vio_disable_interrupts(vscsi->dma_dev);
 261
 262                 state_under_lock = vscsi->new_state;
 263                 flags_under_lock = vscsi->flags;
 264                 vscsi->phyp_acr_state = 0;
 265                 vscsi->phyp_acr_flags = 0;
 266
 267                 spin_unlock_bh(&vscsi->intr_lock);
 268                 rc = ibmvscsis_unregister_command_q(vscsi);
 269                 spin_lock_bh(&vscsi->intr_lock);
 270
 271                 if (state_under_lock != vscsi->new_state)
 272                         vscsi->phyp_acr_state = vscsi->new_state;
 273
 274                 vscsi->phyp_acr_flags = ((~flags_under_lock) & vscsi->flags);
 275
 276                 if (rc == ADAPT_SUCCESS) {
 277                         bytes = vscsi->cmd_q.size * PAGE_SIZE;
 278                         memset(vscsi->cmd_q.base_addr, 0, bytes);
 279                         vscsi->cmd_q.index = 0;
 280                         vscsi->flags |= CRQ_CLOSED;
 281
 282                         ibmvscsis_delete_client_info(vscsi, false);
 283                 }
 284
 285                 dev_dbg(&vscsi->dev, "free_command_q: flags 0x%x, state 0x%hx, acr_flags 0x%x, acr_state 0x%hx\n",
 286                         vscsi->flags, vscsi->state, vscsi->phyp_acr_flags,
 287                         vscsi->phyp_acr_state);
 288         }
 289         return rc;
 290 }
 291
 292 /**
 293  * ibmvscsis_cmd_q_dequeue() - Get valid Command element
 294  * @mask:       Mask to use in case index wraps
 295  * @current_index:      Current index into command queue
 296  * @base_addr:  Pointer to start of command queue
 297  *
 298  * Returns a pointer to a valid command element or NULL, if the command
 299  * queue is empty
 300  *
 301  * EXECUTION ENVIRONMENT:
 302  *      Interrupt environment, interrupt lock held
 303  */
 304 static struct viosrp_crq *ibmvscsis_cmd_q_dequeue(uint mask,
 305                                                   uint *current_index,
 306                                                   struct viosrp_crq *base_addr)
 307 {
 308         struct viosrp_crq *ptr;
 309
 310         ptr = base_addr + *current_index;
 311
 312         if (ptr->valid) {
 313                 *current_index = (*current_index + 1) & mask;
 314                 dma_rmb();
 315         } else {
 316                 ptr = NULL;
 317         }
 318
 319         return ptr;
 320 }
 321
 322 /**
 323  * ibmvscsis_send_init_message() - send initialize message to the client
 324  * @vscsi:      Pointer to our adapter structure
 325  * @format:     Which Init Message format to send
 326  *
 327  * EXECUTION ENVIRONMENT:
 328  *      Interrupt environment interrupt lock held
 329  */
 330 static long ibmvscsis_send_init_message(struct scsi_info *vscsi, u8 format)
 331 {
 332         struct viosrp_crq *crq;
 333         u64 buffer[2] = { 0, 0 };
 334         long rc;
 335
 336         crq = (struct viosrp_crq *)&buffer;
 337         crq->valid = VALID_INIT_MSG;
 338         crq->format = format;
 339         rc = h_send_crq(vscsi->dds.unit_id, cpu_to_be64(buffer[MSG_HI]),
 340                         cpu_to_be64(buffer[MSG_LOW]));
 341
 342         return rc;
 343 }
 344
 345 /**
 346  * ibmvscsis_check_init_msg() - Check init message valid
 347  * @vscsi:      Pointer to our adapter structure
 348  * @format:     Pointer to return format of Init Message, if any.
 349  *              Set to UNUSED_FORMAT if no Init Message in queue.
 350  *
 351  * Checks if an initialize message was queued by the initiatior
 352  * after the queue was created and before the interrupt was enabled.
 353  *
 354  * EXECUTION ENVIRONMENT:
 355  *      Process level only, interrupt lock held
 356  */
 357 static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format)
 358 {
 359         struct viosrp_crq *crq;
 360         long rc = ADAPT_SUCCESS;
 361
 362         crq = ibmvscsis_cmd_q_dequeue(vscsi->cmd_q.mask, &vscsi->cmd_q.index,
 363                                       vscsi->cmd_q.base_addr);
 364         if (!crq) {
 365                 *format = (uint)UNUSED_FORMAT;
 366         } else if (crq->valid == VALID_INIT_MSG && crq->format == INIT_MSG) {
 367                 *format = (uint)INIT_MSG;
 368                 crq->valid = INVALIDATE_CMD_RESP_EL;
 369                 dma_rmb();
 370
 371                 /*
 372                  * the caller has ensured no initialize message was
 373                  * sent after the queue was
 374                  * created so there should be no other message on the queue.
 375                  */
 376                 crq = ibmvscsis_cmd_q_dequeue(vscsi->cmd_q.mask,
 377                                               &vscsi->cmd_q.index,
 378                                               vscsi->cmd_q.base_addr);
 379                 if (crq) {
 380                         *format = (uint)(crq->format);
 381                         rc = ERROR;
 382                         crq->valid = INVALIDATE_CMD_RESP_EL;
 383                         dma_rmb();
 384                 }
 385         } else {
 386                 *format = (uint)(crq->format);
 387                 rc = ERROR;
 388                 crq->valid = INVALIDATE_CMD_RESP_EL;
 389                 dma_rmb();
 390         }
 391
 392         return rc;
 393 }
 394
 395 /**
 396  * ibmvscsis_disconnect() - Helper function to disconnect
 397  * @work:       Pointer to work_struct, gives access to our adapter structure
 398  *
 399  * An error has occurred or the driver received a Transport event,
 400  * and the driver is requesting that the command queue be de-registered
 401  * in a safe manner. If there is no outstanding I/O then we can stop the
 402  * queue. If we are restarting the queue it will be reflected in the
 403  * the state of the adapter.
 404  *
 405  * EXECUTION ENVIRONMENT:
 406  *      Process environment
 407  */
 408 static void ibmvscsis_disconnect(struct work_struct *work)
 409 {
 410         struct scsi_info *vscsi = container_of(work, struct scsi_info,
 411                                                proc_work);
 412         u16 new_state;
 413         bool wait_idle = false;
 414
 415         spin_lock_bh(&vscsi->intr_lock);
 416         new_state = vscsi->new_state;
 417         vscsi->new_state = 0;
 418
 419         vscsi->flags |= DISCONNECT_SCHEDULED;
 420         vscsi->flags &= ~SCHEDULE_DISCONNECT;
 421
 422         dev_dbg(&vscsi->dev, "disconnect: flags 0x%x, state 0x%hx\n",
 423                 vscsi->flags, vscsi->state);
 424
 425         /*
 426          * check which state we are in and see if we
 427          * should transitition to the new state
 428          */
 429         switch (vscsi->state) {
 430         /* Should never be called while in this state. */
 431         case NO_QUEUE:
 432         /*
 433          * Can never transition from this state;
 434          * igonore errors and logout.
 435          */
 436         case UNCONFIGURING:
 437                 break;
 438
 439         /* can transition from this state to UNCONFIGURING */
 440         case ERR_DISCONNECT:
 441                 if (new_state == UNCONFIGURING)
 442                         vscsi->state = new_state;
 443                 break;
 444
 445         /*
 446          * Can transition from this state to to unconfiguring
 447          * or err disconnect.
 448          */
 449         case ERR_DISCONNECT_RECONNECT:
 450                 switch (new_state) {
 451                 case UNCONFIGURING:
 452                 case ERR_DISCONNECT:
 453                         vscsi->state = new_state;
 454                         break;
 455
 456                 case WAIT_IDLE:
 457                         break;
 458                 default:
 459                         break;
 460                 }
 461                 break;
 462
 463         /* can transition from this state to UNCONFIGURING */
 464         case ERR_DISCONNECTED:
 465                 if (new_state == UNCONFIGURING)
 466                         vscsi->state = new_state;
 467                 break;
 468
 469         case WAIT_ENABLED:
 470                 switch (new_state) {
 471                 case UNCONFIGURING:
 472                         vscsi->state = new_state;
 473                         vscsi->flags |= RESPONSE_Q_DOWN;
 474                         vscsi->flags &= ~(SCHEDULE_DISCONNECT |
 475                                           DISCONNECT_SCHEDULED);
 476                         dma_rmb();
 477                         if (vscsi->flags & CFG_SLEEPING) {
 478                                 vscsi->flags &= ~CFG_SLEEPING;
 479                                 complete(&vscsi->unconfig);
 480                         }
 481                         break;
 482
 483                 /* should never happen */
 484                 case ERR_DISCONNECT:
 485                 case ERR_DISCONNECT_RECONNECT:
 486                 case WAIT_IDLE:
 487                         dev_err(&vscsi->dev, "disconnect: invalid state %d for WAIT_IDLE\n",
 488                                 vscsi->state);
 489                         break;
 490                 }
 491                 break;
 492
 493         case WAIT_IDLE:
 494                 switch (new_state) {
 495                 case UNCONFIGURING:
 496                         vscsi->flags |= RESPONSE_Q_DOWN;
 497                         vscsi->state = new_state;
 498                         vscsi->flags &= ~(SCHEDULE_DISCONNECT |
 499                                           DISCONNECT_SCHEDULED);
 500                         ibmvscsis_free_command_q(vscsi);
 501                         break;
 502                 case ERR_DISCONNECT:
 503                 case ERR_DISCONNECT_RECONNECT:
 504                         vscsi->state = new_state;
 505                         break;
 506                 }
 507                 break;
 508
 509         /*
 510          * Initiator has not done a successful srp login
 511          * or has done a successful srp logout ( adapter was not
 512          * busy). In the first case there can be responses queued
 513          * waiting for space on the initiators response queue (MAD)
 514          * The second case the adapter is idle. Assume the worse case,
 515          * i.e. the second case.
 516          */
 517         case WAIT_CONNECTION:
 518         case CONNECTED:
 519         case SRP_PROCESSING:
 520                 wait_idle = true;
 521                 vscsi->state = new_state;
 522                 break;
 523
 524         /* can transition from this state to UNCONFIGURING */
 525         case UNDEFINED:
 526                 if (new_state == UNCONFIGURING)
 527                         vscsi->state = new_state;
 528                 break;
 529         default:
 530                 break;
 531         }
 532
 533         if (wait_idle) {
 534                 dev_dbg(&vscsi->dev, "disconnect start wait, active %d, sched %d\n",
 535                         (int)list_empty(&vscsi->active_q),
 536                         (int)list_empty(&vscsi->schedule_q));
 537                 if (!list_empty(&vscsi->active_q) ||
 538                     !list_empty(&vscsi->schedule_q)) {
 539                         vscsi->flags |= WAIT_FOR_IDLE;
 540                         dev_dbg(&vscsi->dev, "disconnect flags 0x%x\n",
 541                                 vscsi->flags);
 542                         /*
 543                          * This routine is can not be called with the interrupt
 544                          * lock held.
 545                          */
 546                         spin_unlock_bh(&vscsi->intr_lock);
 547                         wait_for_completion(&vscsi->wait_idle);
 548                         spin_lock_bh(&vscsi->intr_lock);
 549                 }
 550                 dev_dbg(&vscsi->dev, "disconnect stop wait\n");
 551
 552                 ibmvscsis_adapter_idle(vscsi);
 553         }
 554
 555         spin_unlock_bh(&vscsi->intr_lock);
 556 }
 557
 558 /**
 559  * ibmvscsis_post_disconnect() - Schedule the disconnect
 560  * @vscsi:      Pointer to our adapter structure
 561  * @new_state:  State to move to after disconnecting
 562  * @flag_bits:  Flags to turn on in adapter structure
 563  *
 564  * If it's already been scheduled, then see if we need to "upgrade"
 565  * the new state (if the one passed in is more "severe" than the
 566  * previous one).
 567  *
 568  * PRECONDITION:
 569  *      interrupt lock is held
 570  */
 571 static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state,
 572                                       uint flag_bits)
 573 {
 574         uint state;
 575
 576         /* check the validity of the new state */
 577         switch (new_state) {
 578         case UNCONFIGURING:
 579         case ERR_DISCONNECT:
 580         case ERR_DISCONNECT_RECONNECT:
 581         case WAIT_IDLE:
 582                 break;
 583
 584         default:
 585                 dev_err(&vscsi->dev, "post_disconnect: Invalid new state %d\n",
 586                         new_state);
 587                 return;
 588         }
 589
 590         vscsi->flags |= flag_bits;
 591
 592         dev_dbg(&vscsi->dev, "post_disconnect: new_state 0x%x, flag_bits 0x%x, vscsi->flags 0x%x, state %hx\n",
 593                 new_state, flag_bits, vscsi->flags, vscsi->state);
 594
 595         if (!(vscsi->flags & (DISCONNECT_SCHEDULED | SCHEDULE_DISCONNECT))) {
 596                 vscsi->flags |= SCHEDULE_DISCONNECT;
 597                 vscsi->new_state = new_state;
 598
 599                 INIT_WORK(&vscsi->proc_work, ibmvscsis_disconnect);
 600                 (void)queue_work(vscsi->work_q, &vscsi->proc_work);
 601         } else {
 602                 if (vscsi->new_state)
 603                         state = vscsi->new_state;
 604                 else
 605                         state = vscsi->state;
 606
 607                 switch (state) {
 608                 case NO_QUEUE:
 609                 case UNCONFIGURING:
 610                         break;
 611
 612                 case ERR_DISCONNECTED:
 613                 case ERR_DISCONNECT:
 614                 case UNDEFINED:
 615                         if (new_state == UNCONFIGURING)
 616                                 vscsi->new_state = new_state;
 617                         break;
 618
 619                 case ERR_DISCONNECT_RECONNECT:
 620                         switch (new_state) {
 621                         case UNCONFIGURING:
 622                         case ERR_DISCONNECT:
 623                                 vscsi->new_state = new_state;
 624                                 break;
 625                         default:
 626                                 break;
 627                         }
 628                         break;
 629
 630                 case WAIT_ENABLED:
 631                 case WAIT_IDLE:
 632                 case WAIT_CONNECTION:
 633                 case CONNECTED:
 634                 case SRP_PROCESSING:
 635                         vscsi->new_state = new_state;
 636                         break;
 637
 638                 default:
 639                         break;
 640                 }
 641         }
 642
 643         dev_dbg(&vscsi->dev, "Leaving post_disconnect: flags 0x%x, new_state 0x%x\n",
 644                 vscsi->flags, vscsi->new_state);
 645 }
 646
 647 /**
 648  * ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message
 649  * @vscsi:      Pointer to our adapter structure
 650  *
 651  * Must be called with interrupt lock held.
 652  */
 653 static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi)
 654 {
 655         long rc = ADAPT_SUCCESS;
 656
 657         switch (vscsi->state) {
 658         case NO_QUEUE:
 659         case ERR_DISCONNECT:
 660         case ERR_DISCONNECT_RECONNECT:
 661         case ERR_DISCONNECTED:
 662         case UNCONFIGURING:
 663         case UNDEFINED:
 664                 rc = ERROR;
 665                 break;
 666
 667         case WAIT_CONNECTION:
 668                 vscsi->state = CONNECTED;
 669                 break;
 670
 671         case WAIT_IDLE:
 672         case SRP_PROCESSING:
 673         case CONNECTED:
 674         case WAIT_ENABLED:
 675         default:
 676                 rc = ERROR;
 677                 dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n",
 678                         vscsi->state);
 679                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
 680                 break;
 681         }
 682
 683         return rc;
 684 }
 685
 686 /**
 687  * ibmvscsis_handle_init_msg() - Respond to an Init Message
 688  * @vscsi:      Pointer to our adapter structure
 689  *
 690  * Must be called with interrupt lock held.
 691  */
 692 static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi)
 693 {
 694         long rc = ADAPT_SUCCESS;
 695
 696         switch (vscsi->state) {
 697         case WAIT_CONNECTION:
 698                 rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG);
 699                 switch (rc) {
 700                 case H_SUCCESS:
 701                         vscsi->state = CONNECTED;
 702                         break;
 703
 704                 case H_PARAMETER:
 705                         dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
 706                                 rc);
 707                         ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
 708                         break;
 709
 710                 case H_DROPPED:
 711                         dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
 712                                 rc);
 713                         rc = ERROR;
 714                         ibmvscsis_post_disconnect(vscsi,
 715                                                   ERR_DISCONNECT_RECONNECT, 0);
 716                         break;
 717
 718                 case H_CLOSED:
 719                         dev_warn(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
 720                                  rc);
 721                         rc = 0;
 722                         break;
 723                 }
 724                 break;
 725
 726         case UNDEFINED:
 727                 rc = ERROR;
 728                 break;
 729
 730         case UNCONFIGURING:
 731                 break;
 732
 733         case WAIT_ENABLED:
 734         case CONNECTED:
 735         case SRP_PROCESSING:
 736         case WAIT_IDLE:
 737         case NO_QUEUE:
 738         case ERR_DISCONNECT:
 739         case ERR_DISCONNECT_RECONNECT:
 740         case ERR_DISCONNECTED:
 741         default:
 742                 rc = ERROR;
 743                 dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n",
 744                         vscsi->state);
 745                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
 746                 break;
 747         }
 748
 749         return rc;
 750 }
 751
 752 /**
 753  * ibmvscsis_init_msg() - Respond to an init message
 754  * @vscsi:      Pointer to our adapter structure
 755  * @crq:        Pointer to CRQ element containing the Init Message
 756  *
 757  * EXECUTION ENVIRONMENT:
 758  *      Interrupt, interrupt lock held
 759  */
 760 static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq)
 761 {
 762         long rc = ADAPT_SUCCESS;
 763
 764         dev_dbg(&vscsi->dev, "init_msg: state 0x%hx\n", vscsi->state);
 765
 766         rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
 767                       (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
 768                       0);
 769         if (rc == H_SUCCESS) {
 770                 vscsi->client_data.partition_number =
 771                         be64_to_cpu(*(u64 *)vscsi->map_buf);
 772                 dev_dbg(&vscsi->dev, "init_msg, part num %d\n",
 773                         vscsi->client_data.partition_number);
 774         } else {
 775                 dev_dbg(&vscsi->dev, "init_msg h_vioctl rc %ld\n", rc);
 776                 rc = ADAPT_SUCCESS;
 777         }
 778
 779         if (crq->format == INIT_MSG) {
 780                 rc = ibmvscsis_handle_init_msg(vscsi);
 781         } else if (crq->format == INIT_COMPLETE_MSG) {
 782                 rc = ibmvscsis_handle_init_compl_msg(vscsi);
 783         } else {
 784                 rc = ERROR;
 785                 dev_err(&vscsi->dev, "init_msg: invalid format %d\n",
 786                         (uint)crq->format);
 787                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
 788         }
 789
 790         return rc;
 791 }
 792
 793 /**
 794  * ibmvscsis_establish_new_q() - Establish new CRQ queue
 795  * @vscsi:      Pointer to our adapter structure
 796  *
 797  * Must be called with interrupt lock held.
 798  */
 799 static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
 800 {
 801         long rc = ADAPT_SUCCESS;
 802         uint format;
 803
 804         rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000,
 805                       0, 0, 0, 0);
 806         if (rc == H_SUCCESS)
 807                 vscsi->flags |= PREP_FOR_SUSPEND_ENABLED;
 808         else if (rc != H_NOT_FOUND)
 809                 dev_err(&vscsi->dev, "Error from Enable Prepare for Suspend: %ld\n",
 810                         rc);
 811
 812         vscsi->flags &= PRESERVE_FLAG_FIELDS;
 813         vscsi->rsp_q_timer.timer_pops = 0;
 814         vscsi->debit = 0;
 815         vscsi->credit = 0;
 816
 817         rc = vio_enable_interrupts(vscsi->dma_dev);
 818         if (rc) {
 819                 dev_warn(&vscsi->dev, "establish_new_q: failed to enable interrupts, rc %ld\n",
 820                          rc);
 821                 return rc;
 822         }
 823
 824         rc = ibmvscsis_check_init_msg(vscsi, &format);
 825         if (rc) {
 826                 dev_err(&vscsi->dev, "establish_new_q: check_init_msg failed, rc %ld\n",
 827                         rc);
 828                 return rc;
 829         }
 830
 831         if (format == UNUSED_FORMAT) {
 832                 rc = ibmvscsis_send_init_message(vscsi, INIT_MSG);
 833                 switch (rc) {
 834                 case H_SUCCESS:
 835                 case H_DROPPED:
 836                 case H_CLOSED:
 837                         rc = ADAPT_SUCCESS;
 838                         break;
 839
 840                 case H_PARAMETER:
 841                 case H_HARDWARE:
 842                         break;
 843
 844                 default:
 845                         vscsi->state = UNDEFINED;
 846                         rc = H_HARDWARE;
 847                         break;
 848                 }
 849         } else if (format == INIT_MSG) {
 850                 rc = ibmvscsis_handle_init_msg(vscsi);
 851         }
 852
 853         return rc;
 854 }
 855
 856 /**
 857  * ibmvscsis_reset_queue() - Reset CRQ Queue
 858  * @vscsi:      Pointer to our adapter structure
 859  *
 860  * This function calls h_free_q and then calls h_reg_q and does all
 861  * of the bookkeeping to get us back to where we can communicate.
 862  *
 863  * Actually, we don't always call h_free_crq.  A problem was discovered
 864  * where one partition would close and reopen his queue, which would
 865  * cause his partner to get a transport event, which would cause him to
 866  * close and reopen his queue, which would cause the original partition
 867  * to get a transport event, etc., etc.  To prevent this, we don't
 868  * actually close our queue if the client initiated the reset, (i.e.
 869  * either we got a transport event or we have detected that the client's
 870  * queue is gone)
 871  *
 872  * EXECUTION ENVIRONMENT:
 873  *      Process environment, called with interrupt lock held
 874  */
 875 static void ibmvscsis_reset_queue(struct scsi_info *vscsi)
 876 {
 877         int bytes;
 878         long rc = ADAPT_SUCCESS;
 879
 880         dev_dbg(&vscsi->dev, "reset_queue: flags 0x%x\n", vscsi->flags);
 881
 882         /* don't reset, the client did it for us */
 883         if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) {
 884                 vscsi->flags &= PRESERVE_FLAG_FIELDS;
 885                 vscsi->rsp_q_timer.timer_pops = 0;
 886                 vscsi->debit = 0;
 887                 vscsi->credit = 0;
 888                 vscsi->state = WAIT_CONNECTION;
 889                 vio_enable_interrupts(vscsi->dma_dev);
 890         } else {
 891                 rc = ibmvscsis_free_command_q(vscsi);
 892                 if (rc == ADAPT_SUCCESS) {
 893                         vscsi->state = WAIT_CONNECTION;
 894
 895                         bytes = vscsi->cmd_q.size * PAGE_SIZE;
 896                         rc = h_reg_crq(vscsi->dds.unit_id,
 897                                        vscsi->cmd_q.crq_token, bytes);
 898                         if (rc == H_CLOSED || rc == H_SUCCESS) {
 899                                 rc = ibmvscsis_establish_new_q(vscsi);
 900                         }
 901
 902                         if (rc != ADAPT_SUCCESS) {
 903                                 dev_dbg(&vscsi->dev, "reset_queue: reg_crq rc %ld\n",
 904                                         rc);
 905
 906                                 vscsi->state = ERR_DISCONNECTED;
 907                                 vscsi->flags |= RESPONSE_Q_DOWN;
 908                                 ibmvscsis_free_command_q(vscsi);
 909                         }
 910                 } else {
 911                         vscsi->state = ERR_DISCONNECTED;
 912                         vscsi->flags |= RESPONSE_Q_DOWN;
 913                 }
 914         }
 915 }
 916
 917 /**
 918  * ibmvscsis_free_cmd_resources() - Free command resources
 919  * @vscsi:      Pointer to our adapter structure
 920  * @cmd:        Command which is not longer in use
 921  *
 922  * Must be called with interrupt lock held.
 923  */
 924 static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
 925                                          struct ibmvscsis_cmd *cmd)
 926 {
 927         struct iu_entry *iue = cmd->iue;
 928
 929         switch (cmd->type) {
 930         case TASK_MANAGEMENT:
 931         case SCSI_CDB:
 932                 /*
 933                  * When the queue goes down this value is cleared, so it
 934                  * cannot be cleared in this general purpose function.
 935                  */
 936                 if (vscsi->debit)
 937                         vscsi->debit -= 1;
 938                 break;
 939         case ADAPTER_MAD:
 940                 vscsi->flags &= ~PROCESSING_MAD;
 941                 break;
 942         case UNSET_TYPE:
 943                 break;
 944         default:
 945                 dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n",
 946                         cmd->type);
 947                 break;
 948         }
 949
 950         cmd->iue = NULL;
 951         list_add_tail(&cmd->list, &vscsi->free_cmd);
 952         srp_iu_put(iue);
 953
 954         if (list_empty(&vscsi->active_q) && list_empty(&vscsi->schedule_q) &&
 955             list_empty(&vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) {
 956                 vscsi->flags &= ~WAIT_FOR_IDLE;
 957                 complete(&vscsi->wait_idle);
 958         }
 959 }
 960
 961 /**
 962  * ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL
 963  * @vscsi:      Pointer to our adapter structure
 964  * @idle:       Indicates whether we were called from adapter_idle.  This
 965  *              is important to know if we need to do a disconnect, since if
 966  *              we're called from adapter_idle, we're still processing the
 967  *              current disconnect, so we can't just call post_disconnect.
 968  *
 969  * This function is called when the adapter is idle when phyp has sent
 970  * us a Prepare for Suspend Transport Event.
 971  *
 972  * EXECUTION ENVIRONMENT:
 973  *      Process or interrupt environment called with interrupt lock held
 974  */
 975 static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle)
 976 {
 977         long rc = 0;
 978         struct viosrp_crq *crq;
 979
 980         /* See if there is a Resume event in the queue */
 981         crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
 982
 983         dev_dbg(&vscsi->dev, "ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n",
 984                 vscsi->flags, vscsi->state, (int)crq->valid);
 985
 986         if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) {
 987                 rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0,
 988                               0, 0);
 989                 if (rc) {
 990                         dev_err(&vscsi->dev, "Ready for Suspend Vioctl failed: %ld\n",
 991                                 rc);
 992                         rc = 0;
 993                 }
 994         } else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) &&
 995                     (vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) ||
 996                    ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
 997                                      (crq->format != RESUME_FROM_SUSP)))) {
 998                 if (idle) {
 999                         vscsi->state = ERR_DISCONNECT_RECONNECT;
1000                         ibmvscsis_reset_queue(vscsi);
1001                         rc = -1;
1002                 } else if (vscsi->state == CONNECTED) {
1003                         ibmvscsis_post_disconnect(vscsi,
1004                                                   ERR_DISCONNECT_RECONNECT, 0);
1005                 }
1006
1007                 vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1008
1009                 if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
1010                                      (crq->format != RESUME_FROM_SUSP)))
1011                         dev_err(&vscsi->dev, "Invalid element in CRQ after Prepare for Suspend");
1012         }
1013
1014         vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED);
1015
1016         return rc;
1017 }
1018
1019 /**
1020  * ibmvscsis_trans_event() - Handle a Transport Event
1021  * @vscsi:      Pointer to our adapter structure
1022  * @crq:        Pointer to CRQ entry containing the Transport Event
1023  *
1024  * Do the logic to close the I_T nexus.  This function may not
1025  * behave to specification.
1026  *
1027  * EXECUTION ENVIRONMENT:
1028  *      Interrupt, interrupt lock held
1029  */
1030 static long ibmvscsis_trans_event(struct scsi_info *vscsi,
1031                                   struct viosrp_crq *crq)
1032 {
1033         long rc = ADAPT_SUCCESS;
1034
1035         dev_dbg(&vscsi->dev, "trans_event: format %d, flags 0x%x, state 0x%hx\n",
1036                 (int)crq->format, vscsi->flags, vscsi->state);
1037
1038         switch (crq->format) {
1039         case MIGRATED:
1040         case PARTNER_FAILED:
1041         case PARTNER_DEREGISTER:
1042                 ibmvscsis_delete_client_info(vscsi, true);
1043                 if (crq->format == MIGRATED)
1044                         vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1045                 switch (vscsi->state) {
1046                 case NO_QUEUE:
1047                 case ERR_DISCONNECTED:
1048                 case UNDEFINED:
1049                         break;
1050
1051                 case UNCONFIGURING:
1052                         vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
1053                         break;
1054
1055                 case WAIT_ENABLED:
1056                         break;
1057
1058                 case WAIT_CONNECTION:
1059                         break;
1060
1061                 case CONNECTED:
1062                         ibmvscsis_post_disconnect(vscsi, WAIT_IDLE,
1063                                                   (RESPONSE_Q_DOWN |
1064                                                    TRANS_EVENT));
1065                         break;
1066
1067                 case SRP_PROCESSING:
1068                         if ((vscsi->debit > 0) ||
1069                             !list_empty(&vscsi->schedule_q) ||
1070                             !list_empty(&vscsi->waiting_rsp) ||
1071                             !list_empty(&vscsi->active_q)) {
1072                                 dev_dbg(&vscsi->dev, "debit %d, sched %d, wait %d, active %d\n",
1073                                         vscsi->debit,
1074                                         (int)list_empty(&vscsi->schedule_q),
1075                                         (int)list_empty(&vscsi->waiting_rsp),
1076                                         (int)list_empty(&vscsi->active_q));
1077                                 dev_warn(&vscsi->dev, "connection lost with outstanding work\n");
1078                         } else {
1079                                 dev_dbg(&vscsi->dev, "trans_event: SRP Processing, but no outstanding work\n");
1080                         }
1081
1082                         ibmvscsis_post_disconnect(vscsi, WAIT_IDLE,
1083                                                   (RESPONSE_Q_DOWN |
1084                                                    TRANS_EVENT));
1085                         break;
1086
1087                 case ERR_DISCONNECT:
1088                 case ERR_DISCONNECT_RECONNECT:
1089                 case WAIT_IDLE:
1090                         vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
1091                         break;
1092                 }
1093                 break;
1094
1095         case PREPARE_FOR_SUSPEND:
1096                 dev_dbg(&vscsi->dev, "Prep for Suspend, crq status = 0x%x\n",
1097                         (int)crq->status);
1098                 switch (vscsi->state) {
1099                 case ERR_DISCONNECTED:
1100                 case WAIT_CONNECTION:
1101                 case CONNECTED:
1102                         ibmvscsis_ready_for_suspend(vscsi, false);
1103                         break;
1104                 case SRP_PROCESSING:
1105                         vscsi->resume_state = vscsi->state;
1106                         vscsi->flags |= PREP_FOR_SUSPEND_PENDING;
1107                         if (crq->status == CRQ_ENTRY_OVERWRITTEN)
1108                                 vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE;
1109                         ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
1110                         break;
1111                 case NO_QUEUE:
1112                 case UNDEFINED:
1113                 case UNCONFIGURING:
1114                 case WAIT_ENABLED:
1115                 case ERR_DISCONNECT:
1116                 case ERR_DISCONNECT_RECONNECT:
1117                 case WAIT_IDLE:
1118                         dev_err(&vscsi->dev, "Invalid state for Prepare for Suspend Trans Event: 0x%x\n",
1119                                 vscsi->state);
1120                         break;
1121                 }
1122                 break;
1123
1124         case RESUME_FROM_SUSP:
1125                 dev_dbg(&vscsi->dev, "Resume from Suspend, crq status = 0x%x\n",
1126                         (int)crq->status);
1127                 if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
1128                         vscsi->flags |= PREP_FOR_SUSPEND_ABORTED;
1129                 } else {
1130                         if ((crq->status == CRQ_ENTRY_OVERWRITTEN) ||
1131                             (vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) {
1132                                 ibmvscsis_post_disconnect(vscsi,
1133                                                           ERR_DISCONNECT_RECONNECT,
1134                                                           0);
1135                                 vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1136                         }
1137                 }
1138                 break;
1139
1140         default:
1141                 rc = ERROR;
1142                 dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
1143                         (uint)crq->format);
1144                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
1145                                           RESPONSE_Q_DOWN);
1146                 break;
1147         }
1148
1149         rc = vscsi->flags & SCHEDULE_DISCONNECT;
1150
1151         dev_dbg(&vscsi->dev, "Leaving trans_event: flags 0x%x, state 0x%hx, rc %ld\n",
1152                 vscsi->flags, vscsi->state, rc);
1153
1154         return rc;
1155 }
1156
1157 /**
1158  * ibmvscsis_poll_cmd_q() - Poll Command Queue
1159  * @vscsi:      Pointer to our adapter structure
1160  *
1161  * Called to handle command elements that may have arrived while
1162  * interrupts were disabled.
1163  *
1164  * EXECUTION ENVIRONMENT:
1165  *      intr_lock must be held
1166  */
1167 static void ibmvscsis_poll_cmd_q(struct scsi_info *vscsi)
1168 {
1169         struct viosrp_crq *crq;
1170         long rc;
1171         bool ack = true;
1172         volatile u8 valid;
1173
1174         dev_dbg(&vscsi->dev, "poll_cmd_q: flags 0x%x, state 0x%hx, q index %ud\n",
1175                 vscsi->flags, vscsi->state, vscsi->cmd_q.index);
1176
1177         rc = vscsi->flags & SCHEDULE_DISCONNECT;
1178         crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
1179         valid = crq->valid;
1180         dma_rmb();
1181
1182         while (valid) {
1183 poll_work:
1184                 vscsi->cmd_q.index =
1185                         (vscsi->cmd_q.index + 1) & vscsi->cmd_q.mask;
1186
1187                 if (!rc) {
1188                         rc = ibmvscsis_parse_command(vscsi, crq);
1189                 } else {
1190                         if ((uint)crq->valid == VALID_TRANS_EVENT) {
1191                                 /*
1192                                  * must service the transport layer events even
1193                                  * in an error state, dont break out until all
1194                                  * the consecutive transport events have been
1195                                  * processed
1196                                  */
1197                                 rc = ibmvscsis_trans_event(vscsi, crq);
1198                         } else if (vscsi->flags & TRANS_EVENT) {
1199                                 /*
1200                                  * if a tranport event has occurred leave
1201                                  * everything but transport events on the queue
1202                                  */
1203                                 dev_dbg(&vscsi->dev, "poll_cmd_q, ignoring\n");
1204
1205                                 /*
1206                                  * need to decrement the queue index so we can
1207                                  * look at the elment again
1208                                  */
1209                                 if (vscsi->cmd_q.index)
1210                                         vscsi->cmd_q.index -= 1;
1211                                 else
1212                                         /*
1213                                          * index is at 0 it just wrapped.
1214                                          * have it index last element in q
1215                                          */
1216                                         vscsi->cmd_q.index = vscsi->cmd_q.mask;
1217                                 break;
1218                         }
1219                 }
1220
1221                 crq->valid = INVALIDATE_CMD_RESP_EL;
1222
1223                 crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
1224                 valid = crq->valid;
1225                 dma_rmb();
1226         }
1227
1228         if (!rc) {
1229                 if (ack) {
1230                         vio_enable_interrupts(vscsi->dma_dev);
1231                         ack = false;
1232                         dev_dbg(&vscsi->dev, "poll_cmd_q, reenabling interrupts\n");
1233                 }
1234                 valid = crq->valid;
1235                 dma_rmb();
1236                 if (valid)
1237                         goto poll_work;
1238         }
1239
1240         dev_dbg(&vscsi->dev, "Leaving poll_cmd_q: rc %ld\n", rc);
1241 }
1242
1243 /**
1244  * ibmvscsis_free_cmd_qs() - Free elements in queue
1245  * @vscsi:      Pointer to our adapter structure
1246  *
1247  * Free all of the elements on all queues that are waiting for
1248  * whatever reason.
1249  *
1250  * PRECONDITION:
1251  *      Called with interrupt lock held
1252  */
1253 static void ibmvscsis_free_cmd_qs(struct scsi_info *vscsi)
1254 {
1255         struct ibmvscsis_cmd *cmd, *nxt;
1256
1257         dev_dbg(&vscsi->dev, "free_cmd_qs: waiting_rsp empty %d, timer starter %d\n",
1258                 (int)list_empty(&vscsi->waiting_rsp),
1259                 vscsi->rsp_q_timer.started);
1260
1261         list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp, list) {
1262                 list_del(&cmd->list);
1263                 ibmvscsis_free_cmd_resources(vscsi, cmd);
1264         }
1265 }
1266
1267 /**
1268  * ibmvscsis_get_free_cmd() - Get free command from list
1269  * @vscsi:      Pointer to our adapter structure
1270  *
1271  * Must be called with interrupt lock held.
1272  */
1273 static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
1274 {
1275         struct ibmvscsis_cmd *cmd = NULL;
1276         struct iu_entry *iue;
1277
1278         iue = srp_iu_get(&vscsi->target);
1279         if (iue) {
1280                 cmd = list_first_entry_or_null(&vscsi->free_cmd,
1281                                                struct ibmvscsis_cmd, list);
1282                 if (cmd) {
1283                         if (cmd->abort_cmd)
1284                                 cmd->abort_cmd = NULL;
1285                         cmd->flags &= ~(DELAY_SEND);
1286                         list_del(&cmd->list);
1287                         cmd->iue = iue;
1288                         cmd->type = UNSET_TYPE;
1289                         memset(&cmd->se_cmd, 0, sizeof(cmd->se_cmd));
1290                 } else {
1291                         srp_iu_put(iue);
1292                 }
1293         }
1294
1295         return cmd;
1296 }
1297
1298 /**
1299  * ibmvscsis_adapter_idle() - Helper function to handle idle adapter
1300  * @vscsi:      Pointer to our adapter structure
1301  *
1302  * This function is called when the adapter is idle when the driver
1303  * is attempting to clear an error condition.
1304  * The adapter is considered busy if any of its cmd queues
1305  * are non-empty. This function can be invoked
1306  * from the off level disconnect function.
1307  *
1308  * EXECUTION ENVIRONMENT:
1309  *      Process environment called with interrupt lock held
1310  */
1311 static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
1312 {
1313         int free_qs = false;
1314         long rc = 0;
1315
1316         dev_dbg(&vscsi->dev, "adapter_idle: flags 0x%x, state 0x%hx\n",
1317                 vscsi->flags, vscsi->state);
1318
1319         /* Only need to free qs if we're disconnecting from client */
1320         if (vscsi->state != WAIT_CONNECTION || vscsi->flags & TRANS_EVENT)
1321                 free_qs = true;
1322
1323         switch (vscsi->state) {
1324         case UNCONFIGURING:
1325                 ibmvscsis_free_command_q(vscsi);
1326                 dma_rmb();
1327                 isync();
1328                 if (vscsi->flags & CFG_SLEEPING) {
1329                         vscsi->flags &= ~CFG_SLEEPING;
1330                         complete(&vscsi->unconfig);
1331                 }
1332                 break;
1333         case ERR_DISCONNECT_RECONNECT:
1334                 ibmvscsis_reset_queue(vscsi);
1335                 dev_dbg(&vscsi->dev, "adapter_idle, disc_rec: flags 0x%x\n",
1336                         vscsi->flags);
1337                 break;
1338
1339         case ERR_DISCONNECT:
1340                 ibmvscsis_free_command_q(vscsi);
1341                 vscsi->flags &= ~(SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED);
1342                 vscsi->flags |= RESPONSE_Q_DOWN;
1343                 if (vscsi->tport.enabled)
1344                         vscsi->state = ERR_DISCONNECTED;
1345                 else
1346                         vscsi->state = WAIT_ENABLED;
1347                 dev_dbg(&vscsi->dev, "adapter_idle, disc: flags 0x%x, state 0x%hx\n",
1348                         vscsi->flags, vscsi->state);
1349                 break;
1350
1351         case WAIT_IDLE:
1352                 vscsi->rsp_q_timer.timer_pops = 0;
1353                 vscsi->debit = 0;
1354                 vscsi->credit = 0;
1355                 if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
1356                         vscsi->state = vscsi->resume_state;
1357                         vscsi->resume_state = 0;
1358                         rc = ibmvscsis_ready_for_suspend(vscsi, true);
1359                         vscsi->flags &= ~DISCONNECT_SCHEDULED;
1360                         if (rc)
1361                                 break;
1362                 } else if (vscsi->flags & TRANS_EVENT) {
1363                         vscsi->state = WAIT_CONNECTION;
1364                         vscsi->flags &= PRESERVE_FLAG_FIELDS;
1365                 } else {
1366                         vscsi->state = CONNECTED;
1367                         vscsi->flags &= ~DISCONNECT_SCHEDULED;
1368                 }
1369
1370                 dev_dbg(&vscsi->dev, "adapter_idle, wait: flags 0x%x, state 0x%hx\n",
1371                         vscsi->flags, vscsi->state);
1372                 ibmvscsis_poll_cmd_q(vscsi);
1373                 break;
1374
1375         case ERR_DISCONNECTED:
1376                 vscsi->flags &= ~DISCONNECT_SCHEDULED;
1377                 dev_dbg(&vscsi->dev, "adapter_idle, disconnected: flags 0x%x, state 0x%hx\n",
1378                         vscsi->flags, vscsi->state);
1379                 break;
1380
1381         default:
1382                 dev_err(&vscsi->dev, "adapter_idle: in invalid state %d\n",
1383                         vscsi->state);
1384                 break;
1385         }
1386
1387         if (free_qs)
1388                 ibmvscsis_free_cmd_qs(vscsi);
1389
1390         /*
1391          * There is a timing window where we could lose a disconnect request.
1392          * The known path to this window occurs during the DISCONNECT_RECONNECT
1393          * case above: reset_queue calls free_command_q, which will release the
1394          * interrupt lock.  During that time, a new post_disconnect call can be
1395          * made with a "more severe" state (DISCONNECT or UNCONFIGURING).
1396          * Because the DISCONNECT_SCHEDULED flag is already set, post_disconnect
1397          * will only set the new_state.  Now free_command_q reacquires the intr
1398          * lock and clears the DISCONNECT_SCHEDULED flag (using PRESERVE_FLAG_
1399          * FIELDS), and the disconnect is lost.  This is particularly bad when
1400          * the new disconnect was for UNCONFIGURING, since the unconfigure hangs
1401          * forever.
1402          * Fix is that free command queue sets acr state and acr flags if there
1403          * is a change under the lock
1404          * note free command queue writes to this state it clears it
1405          * before releasing the lock, different drivers call the free command
1406          * queue different times so dont initialize above
1407          */
1408         if (vscsi->phyp_acr_state != 0) {
1409                 /*
1410                  * set any bits in flags that may have been cleared by
1411                  * a call to free command queue in switch statement
1412                  * or reset queue
1413                  */
1414                 vscsi->flags |= vscsi->phyp_acr_flags;
1415                 ibmvscsis_post_disconnect(vscsi, vscsi->phyp_acr_state, 0);
1416                 vscsi->phyp_acr_state = 0;
1417                 vscsi->phyp_acr_flags = 0;
1418
1419                 dev_dbg(&vscsi->dev, "adapter_idle: flags 0x%x, state 0x%hx, acr_flags 0x%x, acr_state 0x%hx\n",
1420                         vscsi->flags, vscsi->state, vscsi->phyp_acr_flags,
1421                         vscsi->phyp_acr_state);
1422         }
1423
1424         dev_dbg(&vscsi->dev, "Leaving adapter_idle: flags 0x%x, state 0x%hx, new_state 0x%x\n",
1425                 vscsi->flags, vscsi->state, vscsi->new_state);
1426 }
1427
1428 /**
1429  * ibmvscsis_copy_crq_packet() - Copy CRQ Packet
1430  * @vscsi:      Pointer to our adapter structure
1431  * @cmd:        Pointer to command element to use to process the request
1432  * @crq:        Pointer to CRQ entry containing the request
1433  *
1434  * Copy the srp information unit from the hosted
1435  * partition using remote dma
1436  *
1437  * EXECUTION ENVIRONMENT:
1438  *      Interrupt, interrupt lock held
1439  */
1440 static long ibmvscsis_copy_crq_packet(struct scsi_info *vscsi,
1441                                       struct ibmvscsis_cmd *cmd,
1442                                       struct viosrp_crq *crq)
1443 {
1444         struct iu_entry *iue = cmd->iue;
1445         long rc = 0;
1446         u16 len;
1447
1448         len = be16_to_cpu(crq->IU_length);
1449         if ((len > SRP_MAX_IU_LEN) || (len == 0)) {
1450                 dev_err(&vscsi->dev, "copy_crq: Invalid len %d passed", len);
1451                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
1452                 return SRP_VIOLATION;
1453         }
1454
1455         rc = h_copy_rdma(len, vscsi->dds.window[REMOTE].liobn,
1456                          be64_to_cpu(crq->IU_data_ptr),
1457                          vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma);
1458
1459         switch (rc) {
1460         case H_SUCCESS:
1461                 cmd->init_time = mftb();
1462                 iue->remote_token = crq->IU_data_ptr;
1463                 iue->iu_len = len;
1464                 dev_dbg(&vscsi->dev, "copy_crq: ioba 0x%llx, init_time 0x%llx\n",
1465                         be64_to_cpu(crq->IU_data_ptr), cmd->init_time);
1466                 break;
1467         case H_PERMISSION:
1468                 if (connection_broken(vscsi))
1469                         ibmvscsis_post_disconnect(vscsi,
1470                                                   ERR_DISCONNECT_RECONNECT,
1471                                                   (RESPONSE_Q_DOWN |
1472                                                    CLIENT_FAILED));
1473                 else
1474                         ibmvscsis_post_disconnect(vscsi,
1475                                                   ERR_DISCONNECT_RECONNECT, 0);
1476
1477                 dev_err(&vscsi->dev, "copy_crq: h_copy_rdma failed, rc %ld\n",
1478                         rc);
1479                 break;
1480         case H_DEST_PARM:
1481         case H_SOURCE_PARM:
1482         default:
1483                 dev_err(&vscsi->dev, "copy_crq: h_copy_rdma failed, rc %ld\n",
1484                         rc);
1485                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
1486                 break;
1487         }
1488
1489         return rc;
1490 }
1491
1492 /**
1493  * ibmvscsis_adapter_info - Service an Adapter Info MAnagement Data gram
1494  * @vscsi:      Pointer to our adapter structure
1495  * @iue:        Information Unit containing the Adapter Info MAD request
1496  *
1497  * EXECUTION ENVIRONMENT:
1498  *      Interrupt adapter lock is held
1499  */
1500 static long ibmvscsis_adapter_info(struct scsi_info *vscsi,
1501                                    struct iu_entry *iue)
1502 {
1503         struct viosrp_adapter_info *mad = &vio_iu(iue)->mad.adapter_info;
1504         struct mad_adapter_info_data *info;
1505         uint flag_bits = 0;
1506         dma_addr_t token;
1507         long rc;
1508
1509         mad->common.status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1510
1511         if (be16_to_cpu(mad->common.length) > sizeof(*info)) {
1512                 mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1513                 return 0;
1514         }
1515
1516         info = dma_alloc_coherent(&vscsi->dma_dev->dev, sizeof(*info), &token,
1517                                   GFP_ATOMIC);
1518         if (!info) {
1519                 dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n",
1520                         iue->target);
1521                 mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1522                 return 0;
1523         }
1524
1525         /* Get remote info */
1526         rc = h_copy_rdma(be16_to_cpu(mad->common.length),
1527                          vscsi->dds.window[REMOTE].liobn,
1528                          be64_to_cpu(mad->buffer),
1529                          vscsi->dds.window[LOCAL].liobn, token);
1530
1531         if (rc != H_SUCCESS) {
1532                 if (rc == H_PERMISSION) {
1533                         if (connection_broken(vscsi))
1534                                 flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
1535                 }
1536                 dev_warn(&vscsi->dev, "adapter_info: h_copy_rdma from client failed, rc %ld\n",
1537                          rc);
1538                 dev_dbg(&vscsi->dev, "adapter_info: ioba 0x%llx, flags 0x%x, flag_bits 0x%x\n",
1539                         be64_to_cpu(mad->buffer), vscsi->flags, flag_bits);
1540                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
1541                                           flag_bits);
1542                 goto free_dma;
1543         }
1544
1545         /*
1546          * Copy client info, but ignore partition number, which we
1547          * already got from phyp - unless we failed to get it from
1548          * phyp (e.g. if we're running on a p5 system).
1549          */
1550         if (vscsi->client_data.partition_number == 0)
1551                 vscsi->client_data.partition_number =
1552                         be32_to_cpu(info->partition_number);
1553         strncpy(vscsi->client_data.srp_version, info->srp_version,
1554                 sizeof(vscsi->client_data.srp_version));
1555         strncpy(vscsi->client_data.partition_name, info->partition_name,
1556                 sizeof(vscsi->client_data.partition_name));
1557         vscsi->client_data.mad_version = be32_to_cpu(info->mad_version);
1558         vscsi->client_data.os_type = be32_to_cpu(info->os_type);
1559
1560         /* Copy our info */
1561         strncpy(info->srp_version, SRP_VERSION,
1562                 sizeof(info->srp_version));
1563         strncpy(info->partition_name, vscsi->dds.partition_name,
1564                 sizeof(info->partition_name));
1565         info->partition_number = cpu_to_be32(vscsi->dds.partition_num);
1566         info->mad_version = cpu_to_be32(MAD_VERSION_1);
1567         info->os_type = cpu_to_be32(LINUX);
1568         memset(&info->port_max_txu[0], 0, sizeof(info->port_max_txu));
1569         info->port_max_txu[0] = cpu_to_be32(MAX_TXU);
1570
1571         dma_wmb();
1572         rc = h_copy_rdma(sizeof(*info), vscsi->dds.window[LOCAL].liobn,
1573                          token, vscsi->dds.window[REMOTE].liobn,
1574                          be64_to_cpu(mad->buffer));
1575         switch (rc) {
1576         case H_SUCCESS:
1577                 break;
1578
1579         case H_SOURCE_PARM:
1580         case H_DEST_PARM:
1581         case H_PERMISSION:
1582                 if (connection_broken(vscsi))
1583                         flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
1584                 /* Fall through */
1585         default:
1586                 dev_err(&vscsi->dev, "adapter_info: h_copy_rdma to client failed, rc %ld\n",
1587                         rc);
1588                 ibmvscsis_post_disconnect(vscsi,
1589                                           ERR_DISCONNECT_RECONNECT,
1590                                           flag_bits);
1591                 break;
1592         }
1593
1594 free_dma:
1595         dma_free_coherent(&vscsi->dma_dev->dev, sizeof(*info), info, token);
1596         dev_dbg(&vscsi->dev, "Leaving adapter_info, rc %ld\n", rc);
1597
1598         return rc;
1599 }
1600
1601 /**
1602  * ibmvscsis_cap_mad() - Service a Capabilities MAnagement Data gram
1603  * @vscsi:      Pointer to our adapter structure
1604  * @iue:        Information Unit containing the Capabilities MAD request
1605  *
1606  * NOTE: if you return an error from this routine you must be
1607  * disconnecting or you will cause a hang
1608  *
1609  * EXECUTION ENVIRONMENT:
1610  *      Interrupt called with adapter lock held
1611  */
1612 static int ibmvscsis_cap_mad(struct scsi_info *vscsi, struct iu_entry *iue)
1613 {
1614         struct viosrp_capabilities *mad = &vio_iu(iue)->mad.capabilities;
1615         struct capabilities *cap;
1616         struct mad_capability_common *common;
1617         dma_addr_t token;
1618         u16 olen, len, status, min_len, cap_len;
1619         u32 flag;
1620         uint flag_bits = 0;
1621         long rc = 0;
1622
1623         olen = be16_to_cpu(mad->common.length);
1624         /*
1625          * struct capabilities hardcodes a couple capabilities after the
1626          * header, but the capabilities can actually be in any order.
1627          */
1628         min_len = offsetof(struct capabilities, migration);
1629         if ((olen < min_len) || (olen > PAGE_SIZE)) {
1630                 dev_warn(&vscsi->dev, "cap_mad: invalid len %d\n", olen);
1631                 mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1632                 return 0;
1633         }
1634
1635         cap = dma_alloc_coherent(&vscsi->dma_dev->dev, olen, &token,
1636                                  GFP_ATOMIC);
1637         if (!cap) {
1638                 dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n",
1639                         iue->target);
1640                 mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1641                 return 0;
1642         }
1643         rc = h_copy_rdma(olen, vscsi->dds.window[REMOTE].liobn,
1644                          be64_to_cpu(mad->buffer),
1645                          vscsi->dds.window[LOCAL].liobn, token);
1646         if (rc == H_SUCCESS) {
1647                 strncpy(cap->name, dev_name(&vscsi->dma_dev->dev),
1648                         SRP_MAX_LOC_LEN);
1649
1650                 len = olen - min_len;
1651                 status = VIOSRP_MAD_SUCCESS;
1652                 common = (struct mad_capability_common *)&cap->migration;
1653
1654                 while ((len > 0) && (status == VIOSRP_MAD_SUCCESS) && !rc) {
1655                         dev_dbg(&vscsi->dev, "cap_mad: len left %hd, cap type %d, cap len %hd\n",
1656                                 len, be32_to_cpu(common->cap_type),
1657                                 be16_to_cpu(common->length));
1658
1659                         cap_len = be16_to_cpu(common->length);
1660                         if (cap_len > len) {
1661                                 dev_err(&vscsi->dev, "cap_mad: cap len mismatch with total len\n");
1662                                 status = VIOSRP_MAD_FAILED;
1663                                 break;
1664                         }
1665
1666                         if (cap_len == 0) {
1667                                 dev_err(&vscsi->dev, "cap_mad: cap len is 0\n");
1668                                 status = VIOSRP_MAD_FAILED;
1669                                 break;
1670                         }
1671
1672                         switch (common->cap_type) {
1673                         default:
1674                                 dev_dbg(&vscsi->dev, "cap_mad: unsupported capability\n");
1675                                 common->server_support = 0;
1676                                 flag = cpu_to_be32((u32)CAP_LIST_SUPPORTED);
1677                                 cap->flags &= ~flag;
1678                                 break;
1679                         }
1680
1681                         len = len - cap_len;
1682                         common = (struct mad_capability_common *)
1683                                 ((char *)common + cap_len);
1684                 }
1685
1686                 mad->common.status = cpu_to_be16(status);
1687
1688                 dma_wmb();
1689                 rc = h_copy_rdma(olen, vscsi->dds.window[LOCAL].liobn, token,
1690                                  vscsi->dds.window[REMOTE].liobn,
1691                                  be64_to_cpu(mad->buffer));
1692
1693                 if (rc != H_SUCCESS) {
1694                         dev_dbg(&vscsi->dev, "cap_mad: failed to copy to client, rc %ld\n",
1695                                 rc);
1696
1697                         if (rc == H_PERMISSION) {
1698                                 if (connection_broken(vscsi))
1699                                         flag_bits = (RESPONSE_Q_DOWN |
1700                                                      CLIENT_FAILED);
1701                         }
1702
1703                         dev_warn(&vscsi->dev, "cap_mad: error copying data to client, rc %ld\n",
1704                                  rc);
1705                         ibmvscsis_post_disconnect(vscsi,
1706                                                   ERR_DISCONNECT_RECONNECT,
1707                                                   flag_bits);
1708                 }
1709         }
1710
1711         dma_free_coherent(&vscsi->dma_dev->dev, olen, cap, token);
1712
1713         dev_dbg(&vscsi->dev, "Leaving cap_mad, rc %ld, client_cap 0x%x\n",
1714                 rc, vscsi->client_cap);
1715
1716         return rc;
1717 }
1718
1719 /**
1720  * ibmvscsis_process_mad() - Service a MAnagement Data gram
1721  * @vscsi:      Pointer to our adapter structure
1722  * @iue:        Information Unit containing the MAD request
1723  *
1724  * Must be called with interrupt lock held.
1725  */
1726 static long ibmvscsis_process_mad(struct scsi_info *vscsi, struct iu_entry *iue)
1727 {
1728         struct mad_common *mad = (struct mad_common *)&vio_iu(iue)->mad;
1729         struct viosrp_empty_iu *empty;
1730         long rc = ADAPT_SUCCESS;
1731
1732         switch (be32_to_cpu(mad->type)) {
1733         case VIOSRP_EMPTY_IU_TYPE:
1734                 empty = &vio_iu(iue)->mad.empty_iu;
1735                 vscsi->empty_iu_id = be64_to_cpu(empty->buffer);
1736                 vscsi->empty_iu_tag = be64_to_cpu(empty->common.tag);
1737                 mad->status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1738                 break;
1739         case VIOSRP_ADAPTER_INFO_TYPE:
1740                 rc = ibmvscsis_adapter_info(vscsi, iue);
1741                 break;
1742         case VIOSRP_CAPABILITIES_TYPE:
1743                 rc = ibmvscsis_cap_mad(vscsi, iue);
1744                 break;
1745         case VIOSRP_ENABLE_FAST_FAIL:
1746                 if (vscsi->state == CONNECTED) {
1747                         vscsi->fast_fail = true;
1748                         mad->status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1749                 } else {
1750                         dev_warn(&vscsi->dev, "fast fail mad sent after login\n");
1751                         mad->status = cpu_to_be16(VIOSRP_MAD_FAILED);
1752                 }
1753                 break;
1754         default:
1755                 mad->status = cpu_to_be16(VIOSRP_MAD_NOT_SUPPORTED);
1756                 break;
1757         }
1758
1759         return rc;
1760 }
1761
1762 /**
1763  * srp_snd_msg_failed() - Handle an error when sending a response
1764  * @vscsi:      Pointer to our adapter structure
1765  * @rc:         The return code from the h_send_crq command
1766  *
1767  * Must be called with interrupt lock held.
1768  */
1769 static void srp_snd_msg_failed(struct scsi_info *vscsi, long rc)
1770 {
1771         ktime_t kt;
1772
1773         if (rc != H_DROPPED) {
1774                 ibmvscsis_free_cmd_qs(vscsi);
1775
1776                 if (rc == H_CLOSED)
1777                         vscsi->flags |= CLIENT_FAILED;
1778
1779                 /* don't flag the same problem multiple times */
1780                 if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1781                         vscsi->flags |= RESPONSE_Q_DOWN;
1782                         if (!(vscsi->state & (ERR_DISCONNECT |
1783                                               ERR_DISCONNECT_RECONNECT |
1784                                               ERR_DISCONNECTED | UNDEFINED))) {
1785                                 dev_err(&vscsi->dev, "snd_msg_failed: setting RESPONSE_Q_DOWN, state 0x%hx, flags 0x%x, rc %ld\n",
1786                                         vscsi->state, vscsi->flags, rc);
1787                         }
1788                         ibmvscsis_post_disconnect(vscsi,
1789                                                   ERR_DISCONNECT_RECONNECT, 0);
1790                 }
1791                 return;
1792         }
1793
1794         /*
1795          * The response queue is full.
1796          * If the server is processing SRP requests, i.e.
1797          * the client has successfully done an
1798          * SRP_LOGIN, then it will wait forever for room in
1799          * the queue.  However if the system admin
1800          * is attempting to unconfigure the server then one
1801          * or more children will be in a state where
1802          * they are being removed. So if there is even one
1803          * child being removed then the driver assumes
1804          * the system admin is attempting to break the
1805          * connection with the client and MAX_TIMER_POPS
1806          * is honored.
1807          */
1808         if ((vscsi->rsp_q_timer.timer_pops < MAX_TIMER_POPS) ||
1809             (vscsi->state == SRP_PROCESSING)) {
1810                 dev_dbg(&vscsi->dev, "snd_msg_failed: response queue full, flags 0x%x, timer started %d, pops %d\n",
1811                         vscsi->flags, (int)vscsi->rsp_q_timer.started,
1812                         vscsi->rsp_q_timer.timer_pops);
1813
1814                 /*
1815                  * Check if the timer is running; if it
1816                  * is not then start it up.
1817                  */
1818                 if (!vscsi->rsp_q_timer.started) {
1819                         if (vscsi->rsp_q_timer.timer_pops <
1820                             MAX_TIMER_POPS) {
1821                                 kt = WAIT_NANO_SECONDS;
1822                         } else {
1823                                 /*
1824                                  * slide the timeslice if the maximum
1825                                  * timer pops have already happened
1826                                  */
1827                                 kt = ktime_set(WAIT_SECONDS, 0);
1828                         }
1829
1830                         vscsi->rsp_q_timer.started = true;
1831                         hrtimer_start(&vscsi->rsp_q_timer.timer, kt,
1832                                       HRTIMER_MODE_REL);
1833                 }
1834         } else {
1835                 /*
1836                  * TBD: Do we need to worry about this? Need to get
1837                  *      remove working.
1838                  */
1839                 /*
1840                  * waited a long time and it appears the system admin
1841                  * is bring this driver down
1842                  */
1843                 vscsi->flags |= RESPONSE_Q_DOWN;
1844                 ibmvscsis_free_cmd_qs(vscsi);
1845                 /*
1846                  * if the driver is already attempting to disconnect
1847                  * from the client and has already logged an error
1848                  * trace this event but don't put it in the error log
1849                  */
1850                 if (!(vscsi->state & (ERR_DISCONNECT |
1851                                       ERR_DISCONNECT_RECONNECT |
1852                                       ERR_DISCONNECTED | UNDEFINED))) {
1853                         dev_err(&vscsi->dev, "client crq full too long\n");
1854                         ibmvscsis_post_disconnect(vscsi,
1855                                                   ERR_DISCONNECT_RECONNECT,
1856                                                   0);
1857                 }
1858         }
1859 }
1860
1861 /**
1862  * ibmvscsis_send_messages() - Send a Response
1863  * @vscsi:      Pointer to our adapter structure
1864  *
1865  * Send a response, first checking the waiting queue. Responses are
1866  * sent in order they are received. If the response cannot be sent,
1867  * because the client queue is full, it stays on the waiting queue.
1868  *
1869  * PRECONDITION:
1870  *      Called with interrupt lock held
1871  */
1872 static void ibmvscsis_send_messages(struct scsi_info *vscsi)
1873 {
1874         u64 msg_hi = 0;
1875         /* note do not attempt to access the IU_data_ptr with this pointer
1876          * it is not valid
1877          */
1878         struct viosrp_crq *crq = (struct viosrp_crq *)&msg_hi;
1879         struct ibmvscsis_cmd *cmd, *nxt;
1880         struct iu_entry *iue;
1881         long rc = ADAPT_SUCCESS;
1882         bool retry = false;
1883
1884         if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1885                 do {
1886                         retry = false;
1887                         list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp,
1888                                                  list) {
1889                                 /*
1890                                  * Check to make sure abort cmd gets processed
1891                                  * prior to the abort tmr cmd
1892                                  */
1893                                 if (cmd->flags & DELAY_SEND)
1894                                         continue;
1895
1896                                 if (cmd->abort_cmd) {
1897                                         retry = true;
1898                                         cmd->abort_cmd->flags &= ~(DELAY_SEND);
1899                                         cmd->abort_cmd = NULL;
1900                                 }
1901
1902                                 /*
1903                                  * If CMD_T_ABORTED w/o CMD_T_TAS scenarios and
1904                                  * the case where LIO issued a
1905                                  * ABORT_TASK: Sending TMR_TASK_DOES_NOT_EXIST
1906                                  * case then we dont send a response, since it
1907                                  * was already done.
1908                                  */
1909                                 if (cmd->se_cmd.transport_state & CMD_T_ABORTED &&
1910                                     !(cmd->se_cmd.transport_state & CMD_T_TAS)) {
1911                                         list_del(&cmd->list);
1912                                         ibmvscsis_free_cmd_resources(vscsi,
1913                                                                      cmd);
1914                                         /*
1915                                          * With a successfully aborted op
1916                                          * through LIO we want to increment the
1917                                          * the vscsi credit so that when we dont
1918                                          * send a rsp to the original scsi abort
1919                                          * op (h_send_crq), but the tm rsp to
1920                                          * the abort is sent, the credit is
1921                                          * correctly sent with the abort tm rsp.
1922                                          * We would need 1 for the abort tm rsp
1923                                          * and 1 credit for the aborted scsi op.
1924                                          * Thus we need to increment here.
1925                                          * Also we want to increment the credit
1926                                          * here because we want to make sure
1927                                          * cmd is actually released first
1928                                          * otherwise the client will think it
1929                                          * it can send a new cmd, and we could
1930                                          * find ourselves short of cmd elements.
1931                                          */
1932                                         vscsi->credit += 1;
1933                                 } else {
1934                                         iue = cmd->iue;
1935
1936                                         crq->valid = VALID_CMD_RESP_EL;
1937                                         crq->format = cmd->rsp.format;
1938
1939                                         if (cmd->flags & CMD_FAST_FAIL)
1940                                                 crq->status = VIOSRP_ADAPTER_FAIL;
1941
1942                                         crq->IU_length = cpu_to_be16(cmd->rsp.len);
1943
1944                                         rc = h_send_crq(vscsi->dma_dev->unit_address,
1945                                                         be64_to_cpu(msg_hi),
1946                                                         be64_to_cpu(cmd->rsp.tag));
1947
1948                                         dev_dbg(&vscsi->dev, "send_messages: cmd %p, tag 0x%llx, rc %ld\n",
1949                                                 cmd, be64_to_cpu(cmd->rsp.tag),
1950                                                 rc);
1951
1952                                         /* if all ok free up the command
1953                                          * element resources
1954                                          */
1955                                         if (rc == H_SUCCESS) {
1956                                                 /* some movement has occurred */
1957                                                 vscsi->rsp_q_timer.timer_pops = 0;
1958                                                 list_del(&cmd->list);
1959
1960                                                 ibmvscsis_free_cmd_resources(vscsi,
1961                                                                              cmd);
1962                                         } else {
1963                                                 srp_snd_msg_failed(vscsi, rc);
1964                                                 break;
1965                                         }
1966                                 }
1967                         }
1968                 } while (retry);
1969
1970                 if (!rc) {
1971                         /*
1972                          * The timer could pop with the queue empty.  If
1973                          * this happens, rc will always indicate a
1974                          * success; clear the pop count.
1975                          */
1976                         vscsi->rsp_q_timer.timer_pops = 0;
1977                 }
1978         } else {
1979                 ibmvscsis_free_cmd_qs(vscsi);
1980         }
1981 }
1982
1983 /* Called with intr lock held */
1984 static void ibmvscsis_send_mad_resp(struct scsi_info *vscsi,
1985                                     struct ibmvscsis_cmd *cmd,
1986                                     struct viosrp_crq *crq)
1987 {
1988         struct iu_entry *iue = cmd->iue;
1989         struct mad_common *mad = (struct mad_common *)&vio_iu(iue)->mad;
1990         uint flag_bits = 0;
1991         long rc;
1992
1993         dma_wmb();
1994         rc = h_copy_rdma(sizeof(struct mad_common),
1995                          vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma,
1996                          vscsi->dds.window[REMOTE].liobn,
1997                          be64_to_cpu(crq->IU_data_ptr));
1998         if (!rc) {
1999                 cmd->rsp.format = VIOSRP_MAD_FORMAT;
2000                 cmd->rsp.len = sizeof(struct mad_common);
2001                 cmd->rsp.tag = mad->tag;
2002                 list_add_tail(&cmd->list, &vscsi->waiting_rsp);
2003                 ibmvscsis_send_messages(vscsi);
2004         } else {
2005                 dev_dbg(&vscsi->dev, "Error sending mad response, rc %ld\n",
2006                         rc);
2007                 if (rc == H_PERMISSION) {
2008                         if (connection_broken(vscsi))
2009                                 flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
2010                 }
2011                 dev_err(&vscsi->dev, "mad: failed to copy to client, rc %ld\n",
2012                         rc);
2013
2014                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2015                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2016                                           flag_bits);
2017         }
2018 }
2019
2020 /**
2021  * ibmvscsis_mad() - Service a MAnagement Data gram.
2022  * @vscsi:      Pointer to our adapter structure
2023  * @crq:        Pointer to the CRQ entry containing the MAD request
2024  *
2025  * EXECUTION ENVIRONMENT:
2026  *      Interrupt, called with adapter lock held
2027  */
2028 static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq)
2029 {
2030         struct iu_entry *iue;
2031         struct ibmvscsis_cmd *cmd;
2032         struct mad_common *mad;
2033         long rc = ADAPT_SUCCESS;
2034
2035         switch (vscsi->state) {
2036                 /*
2037                  * We have not exchanged Init Msgs yet, so this MAD was sent
2038                  * before the last Transport Event; client will not be
2039                  * expecting a response.
2040                  */
2041         case WAIT_CONNECTION:
2042                 dev_dbg(&vscsi->dev, "mad: in Wait Connection state, ignoring MAD, flags %d\n",
2043                         vscsi->flags);
2044                 return ADAPT_SUCCESS;
2045
2046         case SRP_PROCESSING:
2047         case CONNECTED:
2048                 break;
2049
2050                 /*
2051                  * We should never get here while we're in these states.
2052                  * Just log an error and get out.
2053                  */
2054         case UNCONFIGURING:
2055         case WAIT_IDLE:
2056         case ERR_DISCONNECT:
2057         case ERR_DISCONNECT_RECONNECT:
2058         default:
2059                 dev_err(&vscsi->dev, "mad: invalid adapter state %d for mad\n",
2060                         vscsi->state);
2061                 return ADAPT_SUCCESS;
2062         }
2063
2064         cmd = ibmvscsis_get_free_cmd(vscsi);
2065         if (!cmd) {
2066                 dev_err(&vscsi->dev, "mad: failed to get cmd, debit %d\n",
2067                         vscsi->debit);
2068                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2069                 return ERROR;
2070         }
2071         iue = cmd->iue;
2072         cmd->type = ADAPTER_MAD;
2073
2074         rc = ibmvscsis_copy_crq_packet(vscsi, cmd, crq);
2075         if (!rc) {
2076                 mad = (struct mad_common *)&vio_iu(iue)->mad;
2077
2078                 dev_dbg(&vscsi->dev, "mad: type %d\n", be32_to_cpu(mad->type));
2079
2080                 rc = ibmvscsis_process_mad(vscsi, iue);
2081
2082                 dev_dbg(&vscsi->dev, "mad: status %hd, rc %ld\n",
2083                         be16_to_cpu(mad->status), rc);
2084
2085                 if (!rc)
2086                         ibmvscsis_send_mad_resp(vscsi, cmd, crq);
2087         } else {
2088                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2089         }
2090
2091         dev_dbg(&vscsi->dev, "Leaving mad, rc %ld\n", rc);
2092         return rc;
2093 }
2094
2095 /**
2096  * ibmvscsis_login_rsp() - Create/copy a login response notice to the client
2097  * @vscsi:      Pointer to our adapter structure
2098  * @cmd:        Pointer to the command for the SRP Login request
2099  *
2100  * EXECUTION ENVIRONMENT:
2101  *      Interrupt, interrupt lock held
2102  */
2103 static long ibmvscsis_login_rsp(struct scsi_info *vscsi,
2104                                 struct ibmvscsis_cmd *cmd)
2105 {
2106         struct iu_entry *iue = cmd->iue;
2107         struct srp_login_rsp *rsp = &vio_iu(iue)->srp.login_rsp;
2108         struct format_code *fmt;
2109         uint flag_bits = 0;
2110         long rc = ADAPT_SUCCESS;
2111
2112         memset(rsp, 0, sizeof(struct srp_login_rsp));
2113
2114         rsp->opcode = SRP_LOGIN_RSP;
2115         rsp->req_lim_delta = cpu_to_be32(vscsi->request_limit);
2116         rsp->tag = cmd->rsp.tag;
2117         rsp->max_it_iu_len = cpu_to_be32(SRP_MAX_IU_LEN);
2118         rsp->max_ti_iu_len = cpu_to_be32(SRP_MAX_IU_LEN);
2119         fmt = (struct format_code *)&rsp->buf_fmt;
2120         fmt->buffers = SUPPORTED_FORMATS;
2121         vscsi->credit = 0;
2122
2123         cmd->rsp.len = sizeof(struct srp_login_rsp);
2124
2125         dma_wmb();
2126         rc = h_copy_rdma(cmd->rsp.len, vscsi->dds.window[LOCAL].liobn,
2127                          iue->sbuf->dma, vscsi->dds.window[REMOTE].liobn,
2128                          be64_to_cpu(iue->remote_token));
2129
2130         switch (rc) {
2131         case H_SUCCESS:
2132                 break;
2133
2134         case H_PERMISSION:
2135                 if (connection_broken(vscsi))
2136                         flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
2137                 dev_err(&vscsi->dev, "login_rsp: error copying to client, rc %ld\n",
2138                         rc);
2139                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2140                                           flag_bits);
2141                 break;
2142         case H_SOURCE_PARM:
2143         case H_DEST_PARM:
2144         default:
2145                 dev_err(&vscsi->dev, "login_rsp: error copying to client, rc %ld\n",
2146                         rc);
2147                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2148                 break;
2149         }
2150
2151         return rc;
2152 }
2153
2154 /**
2155  * ibmvscsis_srp_login_rej() - Create/copy a login rejection notice to client
2156  * @vscsi:      Pointer to our adapter structure
2157  * @cmd:        Pointer to the command for the SRP Login request
2158  * @reason:     The reason the SRP Login is being rejected, per SRP protocol
2159  *
2160  * EXECUTION ENVIRONMENT:
2161  *      Interrupt, interrupt lock held
2162  */
2163 static long ibmvscsis_srp_login_rej(struct scsi_info *vscsi,
2164                                     struct ibmvscsis_cmd *cmd, u32 reason)
2165 {
2166         struct iu_entry *iue = cmd->iue;
2167         struct srp_login_rej *rej = &vio_iu(iue)->srp.login_rej;
2168         struct format_code *fmt;
2169         uint flag_bits = 0;
2170         long rc = ADAPT_SUCCESS;
2171
2172         memset(rej, 0, sizeof(*rej));
2173
2174         rej->opcode = SRP_LOGIN_REJ;
2175         rej->reason = cpu_to_be32(reason);
2176         rej->tag = cmd->rsp.tag;
2177         fmt = (struct format_code *)&rej->buf_fmt;
2178         fmt->buffers = SUPPORTED_FORMATS;
2179
2180         cmd->rsp.len = sizeof(*rej);
2181
2182         dma_wmb();
2183         rc = h_copy_rdma(cmd->rsp.len, vscsi->dds.window[LOCAL].liobn,
2184                          iue->sbuf->dma, vscsi->dds.window[REMOTE].liobn,
2185                          be64_to_cpu(iue->remote_token));
2186
2187         switch (rc) {
2188         case H_SUCCESS:
2189                 break;
2190         case H_PERMISSION:
2191                 if (connection_broken(vscsi))
2192                         flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
2193                 dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
2194                         rc);
2195                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2196                                           flag_bits);
2197                 break;
2198         case H_SOURCE_PARM:
2199         case H_DEST_PARM:
2200         default:
2201                 dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
2202                         rc);
2203                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2204                 break;
2205         }
2206
2207         return rc;
2208 }
2209
2210 static int ibmvscsis_make_nexus(struct ibmvscsis_tport *tport)
2211 {
2212         char *name = tport->tport_name;
2213         struct ibmvscsis_nexus *nexus;
2214         struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
2215         int rc;
2216
2217         if (tport->ibmv_nexus) {
2218                 dev_dbg(&vscsi->dev, "tport->ibmv_nexus already exists\n");
2219                 return 0;
2220         }
2221
2222         nexus = kzalloc(sizeof(*nexus), GFP_KERNEL);
2223         if (!nexus) {
2224                 dev_err(&vscsi->dev, "Unable to allocate struct ibmvscsis_nexus\n");
2225                 return -ENOMEM;
2226         }
2227
2228         nexus->se_sess = target_setup_session(&tport->se_tpg, 0, 0,
2229                                               TARGET_PROT_NORMAL, name, nexus,
2230                                               NULL);
2231         if (IS_ERR(nexus->se_sess)) {
2232                 rc = PTR_ERR(nexus->se_sess);
2233                 goto transport_init_fail;
2234         }
2235
2236         tport->ibmv_nexus = nexus;
2237
2238         return 0;
2239
2240 transport_init_fail:
2241         kfree(nexus);
2242         return rc;
2243 }
2244
2245 static int ibmvscsis_drop_nexus(struct ibmvscsis_tport *tport)
2246 {
2247         struct se_session *se_sess;
2248         struct ibmvscsis_nexus *nexus;
2249
2250         nexus = tport->ibmv_nexus;
2251         if (!nexus)
2252                 return -ENODEV;
2253
2254         se_sess = nexus->se_sess;
2255         if (!se_sess)
2256                 return -ENODEV;
2257
2258         /*
2259          * Release the SCSI I_T Nexus to the emulated ibmvscsis Target Port
2260          */
2261         target_remove_session(se_sess);
2262         tport->ibmv_nexus = NULL;
2263         kfree(nexus);
2264
2265         return 0;
2266 }
2267
2268 /**
2269  * ibmvscsis_srp_login() - Process an SRP Login Request
2270  * @vscsi:      Pointer to our adapter structure
2271  * @cmd:        Command element to use to process the SRP Login request
2272  * @crq:        Pointer to CRQ entry containing the SRP Login request
2273  *
2274  * EXECUTION ENVIRONMENT:
2275  *      Interrupt, called with interrupt lock held
2276  */
2277 static long ibmvscsis_srp_login(struct scsi_info *vscsi,
2278                                 struct ibmvscsis_cmd *cmd,
2279                                 struct viosrp_crq *crq)
2280 {
2281         struct iu_entry *iue = cmd->iue;
2282         struct srp_login_req *req = &vio_iu(iue)->srp.login_req;
2283         struct port_id {
2284                 __be64 id_extension;
2285                 __be64 io_guid;
2286         } *iport, *tport;
2287         struct format_code *fmt;
2288         u32 reason = 0x0;
2289         long rc = ADAPT_SUCCESS;
2290
2291         iport = (struct port_id *)req->initiator_port_id;
2292         tport = (struct port_id *)req->target_port_id;
2293         fmt = (struct format_code *)&req->req_buf_fmt;
2294         if (be32_to_cpu(req->req_it_iu_len) > SRP_MAX_IU_LEN)
2295                 reason = SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE;
2296         else if (be32_to_cpu(req->req_it_iu_len) < 64)
2297                 reason = SRP_LOGIN_REJ_UNABLE_ESTABLISH_CHANNEL;
2298         else if ((be64_to_cpu(iport->id_extension) > (MAX_NUM_PORTS - 1)) ||
2299                  (be64_to_cpu(tport->id_extension) > (MAX_NUM_PORTS - 1)))
2300                 reason = SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL;
2301         else if (req->req_flags & SRP_MULTICHAN_MULTI)
2302                 reason = SRP_LOGIN_REJ_MULTI_CHANNEL_UNSUPPORTED;
2303         else if (fmt->buffers & (~SUPPORTED_FORMATS))
2304                 reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
2305         else if ((fmt->buffers & SUPPORTED_FORMATS) == 0)
2306                 reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
2307
2308         if (vscsi->state == SRP_PROCESSING)
2309                 reason = SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED;
2310
2311         rc = ibmvscsis_make_nexus(&vscsi->tport);
2312         if (rc)
2313                 reason = SRP_LOGIN_REJ_UNABLE_ESTABLISH_CHANNEL;
2314
2315         cmd->rsp.format = VIOSRP_SRP_FORMAT;
2316         cmd->rsp.tag = req->tag;
2317
2318         dev_dbg(&vscsi->dev, "srp_login: reason 0x%x\n", reason);
2319
2320         if (reason)
2321                 rc = ibmvscsis_srp_login_rej(vscsi, cmd, reason);
2322         else
2323                 rc = ibmvscsis_login_rsp(vscsi, cmd);
2324
2325         if (!rc) {
2326                 if (!reason)
2327                         vscsi->state = SRP_PROCESSING;
2328
2329                 list_add_tail(&cmd->list, &vscsi->waiting_rsp);
2330                 ibmvscsis_send_messages(vscsi);
2331         } else {
2332                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2333         }
2334
2335         dev_dbg(&vscsi->dev, "Leaving srp_login, rc %ld\n", rc);
2336         return rc;
2337 }
2338
2339 /**
2340  * ibmvscsis_srp_i_logout() - Helper Function to close I_T Nexus
2341  * @vscsi:      Pointer to our adapter structure
2342  * @cmd:        Command element to use to process the Implicit Logout request
2343  * @crq:        Pointer to CRQ entry containing the Implicit Logout request
2344  *
2345  * Do the logic to close the I_T nexus.  This function may not
2346  * behave to specification.
2347  *
2348  * EXECUTION ENVIRONMENT:
2349  *      Interrupt, interrupt lock held
2350  */
2351 static long ibmvscsis_srp_i_logout(struct scsi_info *vscsi,
2352                                    struct ibmvscsis_cmd *cmd,
2353                                    struct viosrp_crq *crq)
2354 {
2355         struct iu_entry *iue = cmd->iue;
2356         struct srp_i_logout *log_out = &vio_iu(iue)->srp.i_logout;
2357         long rc = ADAPT_SUCCESS;
2358
2359         if ((vscsi->debit > 0) || !list_empty(&vscsi->schedule_q) ||
2360             !list_empty(&vscsi->waiting_rsp)) {
2361                 dev_err(&vscsi->dev, "i_logout: outstanding work\n");
2362                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
2363         } else {
2364                 cmd->rsp.format = SRP_FORMAT;
2365                 cmd->rsp.tag = log_out->tag;
2366                 cmd->rsp.len = sizeof(struct mad_common);
2367                 list_add_tail(&cmd->list, &vscsi->waiting_rsp);
2368                 ibmvscsis_send_messages(vscsi);
2369
2370                 ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
2371         }
2372
2373         return rc;
2374 }
2375
2376 /* Called with intr lock held */
2377 static void ibmvscsis_srp_cmd(struct scsi_info *vscsi, struct viosrp_crq *crq)
2378 {
2379         struct ibmvscsis_cmd *cmd;
2380         struct iu_entry *iue;
2381         struct srp_cmd *srp;
2382         struct srp_tsk_mgmt *tsk;
2383         long rc;
2384
2385         if (vscsi->request_limit - vscsi->debit <= 0) {
2386                 /* Client has exceeded request limit */
2387                 dev_err(&vscsi->dev, "Client exceeded the request limit (%d), debit %d\n",
2388                         vscsi->request_limit, vscsi->debit);
2389                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2390                 return;
2391         }
2392
2393         cmd = ibmvscsis_get_free_cmd(vscsi);
2394         if (!cmd) {
2395                 dev_err(&vscsi->dev, "srp_cmd failed to get cmd, debit %d\n",
2396                         vscsi->debit);
2397                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2398                 return;
2399         }
2400         iue = cmd->iue;
2401         srp = &vio_iu(iue)->srp.cmd;
2402
2403         rc = ibmvscsis_copy_crq_packet(vscsi, cmd, crq);
2404         if (rc) {
2405                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2406                 return;
2407         }
2408
2409         if (vscsi->state == SRP_PROCESSING) {
2410                 switch (srp->opcode) {
2411                 case SRP_LOGIN_REQ:
2412                         rc = ibmvscsis_srp_login(vscsi, cmd, crq);
2413                         break;
2414
2415                 case SRP_TSK_MGMT:
2416                         tsk = &vio_iu(iue)->srp.tsk_mgmt;
2417                         dev_dbg(&vscsi->dev, "tsk_mgmt tag: %llu (0x%llx)\n",
2418                                 tsk->tag, tsk->tag);
2419                         cmd->rsp.tag = tsk->tag;
2420                         vscsi->debit += 1;
2421                         cmd->type = TASK_MANAGEMENT;
2422                         list_add_tail(&cmd->list, &vscsi->schedule_q);
2423                         queue_work(vscsi->work_q, &cmd->work);
2424                         break;
2425
2426                 case SRP_CMD:
2427                         dev_dbg(&vscsi->dev, "srp_cmd tag: %llu (0x%llx)\n",
2428                                 srp->tag, srp->tag);
2429                         cmd->rsp.tag = srp->tag;
2430                         vscsi->debit += 1;
2431                         cmd->type = SCSI_CDB;
2432                         /*
2433                          * We want to keep track of work waiting for
2434                          * the workqueue.
2435                          */
2436                         list_add_tail(&cmd->list, &vscsi->schedule_q);
2437                         queue_work(vscsi->work_q, &cmd->work);
2438                         break;
2439
2440                 case SRP_I_LOGOUT:
2441                         rc = ibmvscsis_srp_i_logout(vscsi, cmd, crq);
2442                         break;
2443
2444                 case SRP_CRED_RSP:
2445                 case SRP_AER_RSP:
2446                 default:
2447                         ibmvscsis_free_cmd_resources(vscsi, cmd);
2448                         dev_err(&vscsi->dev, "invalid srp cmd, opcode %d\n",
2449                                 (uint)srp->opcode);
2450                         ibmvscsis_post_disconnect(vscsi,
2451                                                   ERR_DISCONNECT_RECONNECT, 0);
2452                         break;
2453                 }
2454         } else if (srp->opcode == SRP_LOGIN_REQ && vscsi->state == CONNECTED) {
2455                 rc = ibmvscsis_srp_login(vscsi, cmd, crq);
2456         } else {
2457                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2458                 dev_err(&vscsi->dev, "Invalid state %d to handle srp cmd\n",
2459                         vscsi->state);
2460                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2461         }
2462 }
2463
2464 /**
2465  * ibmvscsis_ping_response() - Respond to a ping request
2466  * @vscsi:      Pointer to our adapter structure
2467  *
2468  * Let the client know that the server is alive and waiting on
2469  * its native I/O stack.
2470  * If any type of error occurs from the call to queue a ping
2471  * response then the client is either not accepting or receiving
2472  * interrupts.  Disconnect with an error.
2473  *
2474  * EXECUTION ENVIRONMENT:
2475  *      Interrupt, interrupt lock held
2476  */
2477 static long ibmvscsis_ping_response(struct scsi_info *vscsi)
2478 {
2479         struct viosrp_crq *crq;
2480         u64 buffer[2] = { 0, 0 };
2481         long rc;
2482
2483         crq = (struct viosrp_crq *)&buffer;
2484         crq->valid = VALID_CMD_RESP_EL;
2485         crq->format = (u8)MESSAGE_IN_CRQ;
2486         crq->status = PING_RESPONSE;
2487
2488         rc = h_send_crq(vscsi->dds.unit_id, cpu_to_be64(buffer[MSG_HI]),
2489                         cpu_to_be64(buffer[MSG_LOW]));
2490
2491         switch (rc) {
2492         case H_SUCCESS:
2493                 break;
2494         case H_CLOSED:
2495                 vscsi->flags |= CLIENT_FAILED;
2496                 /* Fall through */
2497         case H_DROPPED:
2498                 vscsi->flags |= RESPONSE_Q_DOWN;
2499                 /* Fall through */
2500         case H_REMOTE_PARM:
2501                 dev_err(&vscsi->dev, "ping_response: h_send_crq failed, rc %ld\n",
2502                         rc);
2503                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2504                 break;
2505         default:
2506                 dev_err(&vscsi->dev, "ping_response: h_send_crq returned unknown rc %ld\n",
2507                         rc);
2508                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
2509                 break;
2510         }
2511
2512         return rc;
2513 }
2514
2515 /**
2516  * ibmvscsis_parse_command() - Parse an element taken from the cmd rsp queue.
2517  * @vscsi:      Pointer to our adapter structure
2518  * @crq:        Pointer to CRQ element containing the SRP request
2519  *
2520  * This function will return success if the command queue element is valid
2521  * and the srp iu or MAD request it pointed to was also valid.  That does
2522  * not mean that an error was not returned to the client.
2523  *
2524  * EXECUTION ENVIRONMENT:
2525  *      Interrupt, intr lock held
2526  */
2527 static long ibmvscsis_parse_command(struct scsi_info *vscsi,
2528                                     struct viosrp_crq *crq)
2529 {
2530         long rc = ADAPT_SUCCESS;
2531
2532         switch (crq->valid) {
2533         case VALID_CMD_RESP_EL:
2534                 switch (crq->format) {
2535                 case OS400_FORMAT:
2536                 case AIX_FORMAT:
2537                 case LINUX_FORMAT:
2538                 case MAD_FORMAT:
2539                         if (vscsi->flags & PROCESSING_MAD) {
2540                                 rc = ERROR;
2541                                 dev_err(&vscsi->dev, "parse_command: already processing mad\n");
2542                                 ibmvscsis_post_disconnect(vscsi,
2543                                                        ERR_DISCONNECT_RECONNECT,
2544                                                        0);
2545                         } else {
2546                                 vscsi->flags |= PROCESSING_MAD;
2547                                 rc = ibmvscsis_mad(vscsi, crq);
2548                         }
2549                         break;
2550
2551                 case SRP_FORMAT:
2552                         ibmvscsis_srp_cmd(vscsi, crq);
2553                         break;
2554
2555                 case MESSAGE_IN_CRQ:
2556                         if (crq->status == PING)
2557                                 ibmvscsis_ping_response(vscsi);
2558                         break;
2559
2560                 default:
2561                         dev_err(&vscsi->dev, "parse_command: invalid format %d\n",
2562                                 (uint)crq->format);
2563                         ibmvscsis_post_disconnect(vscsi,
2564                                                   ERR_DISCONNECT_RECONNECT, 0);
2565                         break;
2566                 }
2567                 break;
2568
2569         case VALID_TRANS_EVENT:
2570                 rc = ibmvscsis_trans_event(vscsi, crq);
2571                 break;
2572
2573         case VALID_INIT_MSG:
2574                 rc = ibmvscsis_init_msg(vscsi, crq);
2575                 break;
2576
2577         default:
2578                 dev_err(&vscsi->dev, "parse_command: invalid valid field %d\n",
2579                         (uint)crq->valid);
2580                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2581                 break;
2582         }
2583
2584         /*
2585          * Return only what the interrupt handler cares
2586          * about. Most errors we keep right on trucking.
2587          */
2588         rc = vscsi->flags & SCHEDULE_DISCONNECT;
2589
2590         return rc;
2591 }
2592
2593 static int read_dma_window(struct scsi_info *vscsi)
2594 {
2595         struct vio_dev *vdev = vscsi->dma_dev;
2596         const __be32 *dma_window;
2597         const __be32 *prop;
2598
2599         /* TODO Using of_parse_dma_window would be better, but it doesn't give
2600          * a way to read multiple windows without already knowing the size of
2601          * a window or the number of windows.
2602          */
2603         dma_window = (const __be32 *)vio_get_attribute(vdev,
2604                                                        "ibm,my-dma-window",
2605                                                        NULL);
2606         if (!dma_window) {
2607                 dev_err(&vscsi->dev, "Couldn't find ibm,my-dma-window property\n");
2608                 return -1;
2609         }
2610
2611         vscsi->dds.window[LOCAL].liobn = be32_to_cpu(*dma_window);
2612         dma_window++;
2613
2614         prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells",
2615                                                  NULL);
2616         if (!prop) {
2617                 dev_warn(&vscsi->dev, "Couldn't find ibm,#dma-address-cells property\n");
2618                 dma_window++;
2619         } else {
2620                 dma_window += be32_to_cpu(*prop);
2621         }
2622
2623         prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells",
2624                                                  NULL);
2625         if (!prop) {
2626                 dev_warn(&vscsi->dev, "Couldn't find ibm,#dma-size-cells property\n");
2627                 dma_window++;
2628         } else {
2629                 dma_window += be32_to_cpu(*prop);
2630         }
2631
2632         /* dma_window should point to the second window now */
2633         vscsi->dds.window[REMOTE].liobn = be32_to_cpu(*dma_window);
2634
2635         return 0;
2636 }
2637
2638 static struct ibmvscsis_tport *ibmvscsis_lookup_port(const char *name)
2639 {
2640         struct ibmvscsis_tport *tport = NULL;
2641         struct vio_dev *vdev;
2642         struct scsi_info *vscsi;
2643
2644         spin_lock_bh(&ibmvscsis_dev_lock);
2645         list_for_each_entry(vscsi, &ibmvscsis_dev_list, list) {
2646                 vdev = vscsi->dma_dev;
2647                 if (!strcmp(dev_name(&vdev->dev), name)) {
2648                         tport = &vscsi->tport;
2649                         break;
2650                 }
2651         }
2652         spin_unlock_bh(&ibmvscsis_dev_lock);
2653
2654         return tport;
2655 }
2656
2657 /**
2658  * ibmvscsis_parse_cmd() - Parse SRP Command
2659  * @vscsi:      Pointer to our adapter structure
2660  * @cmd:        Pointer to command element with SRP command
2661  *
2662  * Parse the srp command; if it is valid then submit it to tcm.
2663  * Note: The return code does not reflect the status of the SCSI CDB.
2664  *
2665  * EXECUTION ENVIRONMENT:
2666  *      Process level
2667  */
2668 static void ibmvscsis_parse_cmd(struct scsi_info *vscsi,
2669                                 struct ibmvscsis_cmd *cmd)
2670 {
2671         struct iu_entry *iue = cmd->iue;
2672         struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
2673         struct ibmvscsis_nexus *nexus;
2674         u64 data_len = 0;
2675         enum dma_data_direction dir;
2676         int attr = 0;
2677         int rc = 0;
2678
2679         nexus = vscsi->tport.ibmv_nexus;
2680         /*
2681          * additional length in bytes.  Note that the SRP spec says that
2682          * additional length is in 4-byte words, but technically the
2683          * additional length field is only the upper 6 bits of the byte.
2684          * The lower 2 bits are reserved.  If the lower 2 bits are 0 (as
2685          * all reserved fields should be), then interpreting the byte as
2686          * an int will yield the length in bytes.
2687          */
2688         if (srp->add_cdb_len & 0x03) {
2689                 dev_err(&vscsi->dev, "parse_cmd: reserved bits set in IU\n");
2690                 spin_lock_bh(&vscsi->intr_lock);
2691                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2692                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2693                 spin_unlock_bh(&vscsi->intr_lock);
2694                 return;
2695         }
2696
2697         if (srp_get_desc_table(srp, &dir, &data_len)) {
2698                 dev_err(&vscsi->dev, "0x%llx: parsing SRP descriptor table failed.\n",
2699                         srp->tag);
2700                 goto fail;
2701         }
2702
2703         cmd->rsp.sol_not = srp->sol_not;
2704
2705         switch (srp->task_attr) {
2706         case SRP_SIMPLE_TASK:
2707                 attr = TCM_SIMPLE_TAG;
2708                 break;
2709         case SRP_ORDERED_TASK:
2710                 attr = TCM_ORDERED_TAG;
2711                 break;
2712         case SRP_HEAD_TASK:
2713                 attr = TCM_HEAD_TAG;
2714                 break;
2715         case SRP_ACA_TASK:
2716                 attr = TCM_ACA_TAG;
2717                 break;
2718         default:
2719                 dev_err(&vscsi->dev, "Invalid task attribute %d\n",
2720                         srp->task_attr);
2721                 goto fail;
2722         }
2723
2724         cmd->se_cmd.tag = be64_to_cpu(srp->tag);
2725
2726         spin_lock_bh(&vscsi->intr_lock);
2727         list_add_tail(&cmd->list, &vscsi->active_q);
2728         spin_unlock_bh(&vscsi->intr_lock);
2729
2730         srp->lun.scsi_lun[0] &= 0x3f;
2731
2732         rc = target_submit_cmd(&cmd->se_cmd, nexus->se_sess, srp->cdb,
2733                                cmd->sense_buf, scsilun_to_int(&srp->lun),
2734                                data_len, attr, dir, 0);
2735         if (rc) {
2736                 dev_err(&vscsi->dev, "target_submit_cmd failed, rc %d\n", rc);
2737                 spin_lock_bh(&vscsi->intr_lock);
2738                 list_del(&cmd->list);
2739                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2740                 spin_unlock_bh(&vscsi->intr_lock);
2741                 goto fail;
2742         }
2743         return;
2744
2745 fail:
2746         spin_lock_bh(&vscsi->intr_lock);
2747         ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2748         spin_unlock_bh(&vscsi->intr_lock);
2749 }
2750
2751 /**
2752  * ibmvscsis_parse_task() - Parse SRP Task Management Request
2753  * @vscsi:      Pointer to our adapter structure
2754  * @cmd:        Pointer to command element with SRP task management request
2755  *
2756  * Parse the srp task management request; if it is valid then submit it to tcm.
2757  * Note: The return code does not reflect the status of the task management
2758  * request.
2759  *
2760  * EXECUTION ENVIRONMENT:
2761  *      Processor level
2762  */
2763 static void ibmvscsis_parse_task(struct scsi_info *vscsi,
2764                                  struct ibmvscsis_cmd *cmd)
2765 {
2766         struct iu_entry *iue = cmd->iue;
2767         struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
2768         int tcm_type;
2769         u64 tag_to_abort = 0;
2770         int rc = 0;
2771         struct ibmvscsis_nexus *nexus;
2772
2773         nexus = vscsi->tport.ibmv_nexus;
2774
2775         cmd->rsp.sol_not = srp_tsk->sol_not;
2776
2777         switch (srp_tsk->tsk_mgmt_func) {
2778         case SRP_TSK_ABORT_TASK:
2779                 tcm_type = TMR_ABORT_TASK;
2780                 tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
2781                 break;
2782         case SRP_TSK_ABORT_TASK_SET:
2783                 tcm_type = TMR_ABORT_TASK_SET;
2784                 break;
2785         case SRP_TSK_CLEAR_TASK_SET:
2786                 tcm_type = TMR_CLEAR_TASK_SET;
2787                 break;
2788         case SRP_TSK_LUN_RESET:
2789                 tcm_type = TMR_LUN_RESET;
2790                 break;
2791         case SRP_TSK_CLEAR_ACA:
2792                 tcm_type = TMR_CLEAR_ACA;
2793                 break;
2794         default:
2795                 dev_err(&vscsi->dev, "unknown task mgmt func %d\n",
2796                         srp_tsk->tsk_mgmt_func);
2797                 cmd->se_cmd.se_tmr_req->response =
2798                         TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
2799                 rc = -1;
2800                 break;
2801         }
2802
2803         if (!rc) {
2804                 cmd->se_cmd.tag = be64_to_cpu(srp_tsk->tag);
2805
2806                 spin_lock_bh(&vscsi->intr_lock);
2807                 list_add_tail(&cmd->list, &vscsi->active_q);
2808                 spin_unlock_bh(&vscsi->intr_lock);
2809
2810                 srp_tsk->lun.scsi_lun[0] &= 0x3f;
2811
2812                 dev_dbg(&vscsi->dev, "calling submit_tmr, func %d\n",
2813                         srp_tsk->tsk_mgmt_func);
2814                 rc = target_submit_tmr(&cmd->se_cmd, nexus->se_sess, NULL,
2815                                        scsilun_to_int(&srp_tsk->lun), srp_tsk,
2816                                        tcm_type, GFP_KERNEL, tag_to_abort, 0);
2817                 if (rc) {
2818                         dev_err(&vscsi->dev, "target_submit_tmr failed, rc %d\n",
2819                                 rc);
2820                         spin_lock_bh(&vscsi->intr_lock);
2821                         list_del(&cmd->list);
2822                         spin_unlock_bh(&vscsi->intr_lock);
2823                         cmd->se_cmd.se_tmr_req->response =
2824                                 TMR_FUNCTION_REJECTED;
2825                 }
2826         }
2827
2828         if (rc)
2829                 transport_send_check_condition_and_sense(&cmd->se_cmd, 0, 0);
2830 }
2831
2832 static void ibmvscsis_scheduler(struct work_struct *work)
2833 {
2834         struct ibmvscsis_cmd *cmd = container_of(work, struct ibmvscsis_cmd,
2835                                                  work);
2836         struct scsi_info *vscsi = cmd->adapter;
2837
2838         spin_lock_bh(&vscsi->intr_lock);
2839
2840         /* Remove from schedule_q */
2841         list_del(&cmd->list);
2842
2843         /* Don't submit cmd if we're disconnecting */
2844         if (vscsi->flags & (SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED)) {
2845                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2846
2847                 /* ibmvscsis_disconnect might be waiting for us */
2848                 if (list_empty(&vscsi->active_q) &&
2849                     list_empty(&vscsi->schedule_q) &&
2850                     (vscsi->flags & WAIT_FOR_IDLE)) {
2851                         vscsi->flags &= ~WAIT_FOR_IDLE;
2852                         complete(&vscsi->wait_idle);
2853                 }
2854
2855                 spin_unlock_bh(&vscsi->intr_lock);
2856                 return;
2857         }
2858
2859         spin_unlock_bh(&vscsi->intr_lock);
2860
2861         switch (cmd->type) {
2862         case SCSI_CDB:
2863                 ibmvscsis_parse_cmd(vscsi, cmd);
2864                 break;
2865         case TASK_MANAGEMENT:
2866                 ibmvscsis_parse_task(vscsi, cmd);
2867                 break;
2868         default:
2869                 dev_err(&vscsi->dev, "scheduler, invalid cmd type %d\n",
2870                         cmd->type);
2871                 spin_lock_bh(&vscsi->intr_lock);
2872                 ibmvscsis_free_cmd_resources(vscsi, cmd);
2873                 spin_unlock_bh(&vscsi->intr_lock);
2874                 break;
2875         }
2876 }
2877
2878 static int ibmvscsis_alloc_cmds(struct scsi_info *vscsi, int num)
2879 {
2880         struct ibmvscsis_cmd *cmd;
2881         int i;
2882
2883         INIT_LIST_HEAD(&vscsi->free_cmd);
2884         vscsi->cmd_pool = kcalloc(num, sizeof(struct ibmvscsis_cmd),
2885                                   GFP_KERNEL);
2886         if (!vscsi->cmd_pool)
2887                 return -ENOMEM;
2888
2889         for (i = 0, cmd = (struct ibmvscsis_cmd *)vscsi->cmd_pool; i < num;
2890              i++, cmd++) {
2891                 cmd->abort_cmd = NULL;
2892                 cmd->adapter = vscsi;
2893                 INIT_WORK(&cmd->work, ibmvscsis_scheduler);
2894                 list_add_tail(&cmd->list, &vscsi->free_cmd);
2895         }
2896
2897         return 0;
2898 }
2899
2900 static void ibmvscsis_free_cmds(struct scsi_info *vscsi)
2901 {
2902         kfree(vscsi->cmd_pool);
2903         vscsi->cmd_pool = NULL;
2904         INIT_LIST_HEAD(&vscsi->free_cmd);
2905 }
2906
2907 /**
2908  * ibmvscsis_service_wait_q() - Service Waiting Queue
2909  * @timer:      Pointer to timer which has expired
2910  *
2911  * This routine is called when the timer pops to service the waiting
2912  * queue. Elements on the queue have completed, their responses have been
2913  * copied to the client, but the client's response queue was full so
2914  * the queue message could not be sent. The routine grabs the proper locks
2915  * and calls send messages.
2916  *
2917  * EXECUTION ENVIRONMENT:
2918  *      called at interrupt level
2919  */
2920 static enum hrtimer_restart ibmvscsis_service_wait_q(struct hrtimer *timer)
2921 {
2922         struct timer_cb *p_timer = container_of(timer, struct timer_cb, timer);
2923         struct scsi_info *vscsi = container_of(p_timer, struct scsi_info,
2924                                                rsp_q_timer);
2925
2926         spin_lock_bh(&vscsi->intr_lock);
2927         p_timer->timer_pops += 1;
2928         p_timer->started = false;
2929         ibmvscsis_send_messages(vscsi);
2930         spin_unlock_bh(&vscsi->intr_lock);
2931
2932         return HRTIMER_NORESTART;
2933 }
2934
2935 static long ibmvscsis_alloctimer(struct scsi_info *vscsi)
2936 {
2937         struct timer_cb *p_timer;
2938
2939         p_timer = &vscsi->rsp_q_timer;
2940         hrtimer_init(&p_timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2941
2942         p_timer->timer.function = ibmvscsis_service_wait_q;
2943         p_timer->started = false;
2944         p_timer->timer_pops = 0;
2945
2946         return ADAPT_SUCCESS;
2947 }
2948
2949 static void ibmvscsis_freetimer(struct scsi_info *vscsi)
2950 {
2951         struct timer_cb *p_timer;
2952
2953         p_timer = &vscsi->rsp_q_timer;
2954
2955         (void)hrtimer_cancel(&p_timer->timer);
2956
2957         p_timer->started = false;
2958         p_timer->timer_pops = 0;
2959 }
2960
2961 static irqreturn_t ibmvscsis_interrupt(int dummy, void *data)
2962 {
2963         struct scsi_info *vscsi = data;
2964
2965         vio_disable_interrupts(vscsi->dma_dev);
2966         tasklet_schedule(&vscsi->work_task);
2967
2968         return IRQ_HANDLED;
2969 }
2970
2971 /**
2972  * ibmvscsis_enable_change_state() - Set new state based on enabled status
2973  * @vscsi:      Pointer to our adapter structure
2974  *
2975  * This function determines our new state now that we are enabled.  This
2976  * may involve sending an Init Complete message to the client.
2977  *
2978  * Must be called with interrupt lock held.
2979  */
2980 static long ibmvscsis_enable_change_state(struct scsi_info *vscsi)
2981 {
2982         int bytes;
2983         long rc = ADAPT_SUCCESS;
2984
2985         bytes = vscsi->cmd_q.size * PAGE_SIZE;
2986         rc = h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, bytes);
2987         if (rc == H_CLOSED || rc == H_SUCCESS) {
2988                 vscsi->state = WAIT_CONNECTION;
2989                 rc = ibmvscsis_establish_new_q(vscsi);
2990         }
2991
2992         if (rc != ADAPT_SUCCESS) {
2993                 vscsi->state = ERR_DISCONNECTED;
2994                 vscsi->flags |= RESPONSE_Q_DOWN;
2995         }
2996
2997         return rc;
2998 }
2999
3000 /**
3001  * ibmvscsis_create_command_q() - Create Command Queue
3002  * @vscsi:      Pointer to our adapter structure
3003  * @num_cmds:   Currently unused.  In the future, may be used to determine
3004  *              the size of the CRQ.
3005  *
3006  * Allocates memory for command queue maps remote memory into an ioba
3007  * initializes the command response queue
3008  *
3009  * EXECUTION ENVIRONMENT:
3010  *      Process level only
3011  */
3012 static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds)
3013 {
3014         int pages;
3015         struct vio_dev *vdev = vscsi->dma_dev;
3016
3017         /* We might support multiple pages in the future, but just 1 for now */
3018         pages = 1;
3019
3020         vscsi->cmd_q.size = pages;
3021
3022         vscsi->cmd_q.base_addr =
3023                 (struct viosrp_crq *)get_zeroed_page(GFP_KERNEL);
3024         if (!vscsi->cmd_q.base_addr)
3025                 return -ENOMEM;
3026
3027         vscsi->cmd_q.mask = ((uint)pages * CRQ_PER_PAGE) - 1;
3028
3029         vscsi->cmd_q.crq_token = dma_map_single(&vdev->dev,
3030                                                 vscsi->cmd_q.base_addr,
3031                                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3032         if (dma_mapping_error(&vdev->dev, vscsi->cmd_q.crq_token)) {
3033                 free_page((unsigned long)vscsi->cmd_q.base_addr);
3034                 return -ENOMEM;
3035         }
3036
3037         return 0;
3038 }
3039
3040 /**
3041  * ibmvscsis_destroy_command_q - Destroy Command Queue
3042  * @vscsi:      Pointer to our adapter structure
3043  *
3044  * Releases memory for command queue and unmaps mapped remote memory.
3045  *
3046  * EXECUTION ENVIRONMENT:
3047  *      Process level only
3048  */
3049 static void ibmvscsis_destroy_command_q(struct scsi_info *vscsi)
3050 {
3051         dma_unmap_single(&vscsi->dma_dev->dev, vscsi->cmd_q.crq_token,
3052                          PAGE_SIZE, DMA_BIDIRECTIONAL);
3053         free_page((unsigned long)vscsi->cmd_q.base_addr);
3054         vscsi->cmd_q.base_addr = NULL;
3055         vscsi->state = NO_QUEUE;
3056 }
3057
3058 static u8 ibmvscsis_fast_fail(struct scsi_info *vscsi,
3059                               struct ibmvscsis_cmd *cmd)
3060 {
3061         struct iu_entry *iue = cmd->iue;
3062         struct se_cmd *se_cmd = &cmd->se_cmd;
3063         struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
3064         struct scsi_sense_hdr sshdr;
3065         u8 rc = se_cmd->scsi_status;
3066
3067         if (vscsi->fast_fail && (READ_CMD(srp->cdb) || WRITE_CMD(srp->cdb)))
3068                 if (scsi_normalize_sense(se_cmd->sense_buffer,
3069                                          se_cmd->scsi_sense_length, &sshdr))
3070                         if (sshdr.sense_key == HARDWARE_ERROR &&
3071                             (se_cmd->residual_count == 0 ||
3072                              se_cmd->residual_count == se_cmd->data_length)) {
3073                                 rc = NO_SENSE;
3074                                 cmd->flags |= CMD_FAST_FAIL;
3075                         }
3076
3077         return rc;
3078 }
3079
3080 /**
3081  * srp_build_response() - Build an SRP response buffer
3082  * @vscsi:      Pointer to our adapter structure
3083  * @cmd:        Pointer to command for which to send the response
3084  * @len_p:      Where to return the length of the IU response sent.  This
3085  *              is needed to construct the CRQ response.
3086  *
3087  * Build the SRP response buffer and copy it to the client's memory space.
3088  */
3089 static long srp_build_response(struct scsi_info *vscsi,
3090                                struct ibmvscsis_cmd *cmd, uint *len_p)
3091 {
3092         struct iu_entry *iue = cmd->iue;
3093         struct se_cmd *se_cmd = &cmd->se_cmd;
3094         struct srp_rsp *rsp;
3095         uint len;
3096         u32 rsp_code;
3097         char *data;
3098         u32 *tsk_status;
3099         long rc = ADAPT_SUCCESS;
3100
3101         spin_lock_bh(&vscsi->intr_lock);
3102
3103         rsp = &vio_iu(iue)->srp.rsp;
3104         len = sizeof(*rsp);
3105         memset(rsp, 0, len);
3106         data = rsp->data;
3107
3108         rsp->opcode = SRP_RSP;
3109
3110         rsp->req_lim_delta = cpu_to_be32(1 + vscsi->credit);
3111         rsp->tag = cmd->rsp.tag;
3112         rsp->flags = 0;
3113
3114         if (cmd->type == SCSI_CDB) {
3115                 rsp->status = ibmvscsis_fast_fail(vscsi, cmd);
3116                 if (rsp->status) {
3117                         dev_dbg(&vscsi->dev, "build_resp: cmd %p, scsi status %d\n",
3118                                 cmd, (int)rsp->status);
3119                         ibmvscsis_determine_resid(se_cmd, rsp);
3120                         if (se_cmd->scsi_sense_length && se_cmd->sense_buffer) {
3121                                 rsp->sense_data_len =
3122                                         cpu_to_be32(se_cmd->scsi_sense_length);
3123                                 rsp->flags |= SRP_RSP_FLAG_SNSVALID;
3124                                 len += se_cmd->scsi_sense_length;
3125                                 memcpy(data, se_cmd->sense_buffer,
3126                                        se_cmd->scsi_sense_length);
3127                         }
3128                         rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3129                                 UCSOLNT_RESP_SHIFT;
3130                 } else if (cmd->flags & CMD_FAST_FAIL) {
3131                         dev_dbg(&vscsi->dev, "build_resp: cmd %p, fast fail\n",
3132                                 cmd);
3133                         rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3134                                 UCSOLNT_RESP_SHIFT;
3135                 } else {
3136                         rsp->sol_not = (cmd->rsp.sol_not & SCSOLNT) >>
3137                                 SCSOLNT_RESP_SHIFT;
3138                 }
3139         } else {
3140                 /* this is task management */
3141                 rsp->status = 0;
3142                 rsp->resp_data_len = cpu_to_be32(4);
3143                 rsp->flags |= SRP_RSP_FLAG_RSPVALID;
3144
3145                 switch (se_cmd->se_tmr_req->response) {
3146                 case TMR_FUNCTION_COMPLETE:
3147                 case TMR_TASK_DOES_NOT_EXIST:
3148                         rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_COMPLETE;
3149                         rsp->sol_not = (cmd->rsp.sol_not & SCSOLNT) >>
3150                                 SCSOLNT_RESP_SHIFT;
3151                         break;
3152                 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3153                 case TMR_LUN_DOES_NOT_EXIST:
3154                         rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_NOT_SUPPORTED;
3155                         rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3156                                 UCSOLNT_RESP_SHIFT;
3157                         break;
3158                 case TMR_FUNCTION_FAILED:
3159                 case TMR_FUNCTION_REJECTED:
3160                 default:
3161                         rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_FAILED;
3162                         rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3163                                 UCSOLNT_RESP_SHIFT;
3164                         break;
3165                 }
3166
3167                 tsk_status = (u32 *)data;
3168                 *tsk_status = cpu_to_be32(rsp_code);
3169                 data = (char *)(tsk_status + 1);
3170                 len += 4;
3171         }
3172
3173         dma_wmb();
3174         rc = h_copy_rdma(len, vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma,
3175                          vscsi->dds.window[REMOTE].liobn,
3176                          be64_to_cpu(iue->remote_token));
3177
3178         switch (rc) {
3179         case H_SUCCESS:
3180                 vscsi->credit = 0;
3181                 *len_p = len;
3182                 break;
3183         case H_PERMISSION:
3184                 if (connection_broken(vscsi))
3185                         vscsi->flags |= RESPONSE_Q_DOWN | CLIENT_FAILED;
3186
3187                 dev_err(&vscsi->dev, "build_response: error copying to client, rc %ld, flags 0x%x, state 0x%hx\n",
3188                         rc, vscsi->flags, vscsi->state);
3189                 break;
3190         case H_SOURCE_PARM:
3191         case H_DEST_PARM:
3192         default:
3193                 dev_err(&vscsi->dev, "build_response: error copying to client, rc %ld\n",
3194                         rc);
3195                 break;
3196         }
3197
3198         spin_unlock_bh(&vscsi->intr_lock);
3199
3200         return rc;
3201 }
3202
3203 static int ibmvscsis_rdma(struct ibmvscsis_cmd *cmd, struct scatterlist *sg,
3204                           int nsg, struct srp_direct_buf *md, int nmd,
3205                           enum dma_data_direction dir, unsigned int bytes)
3206 {
3207         struct iu_entry *iue = cmd->iue;
3208         struct srp_target *target = iue->target;
3209         struct scsi_info *vscsi = target->ldata;
3210         struct scatterlist *sgp;
3211         dma_addr_t client_ioba, server_ioba;
3212         ulong buf_len;
3213         ulong client_len, server_len;
3214         int md_idx;
3215         long tx_len;
3216         long rc = 0;
3217
3218         if (bytes == 0)
3219                 return 0;
3220
3221         sgp = sg;
3222         client_len = 0;
3223         server_len = 0;
3224         md_idx = 0;
3225         tx_len = bytes;
3226
3227         do {
3228                 if (client_len == 0) {
3229                         if (md_idx >= nmd) {
3230                                 dev_err(&vscsi->dev, "rdma: ran out of client memory descriptors\n");
3231                                 rc = -EIO;
3232                                 break;
3233                         }
3234                         client_ioba = be64_to_cpu(md[md_idx].va);
3235                         client_len = be32_to_cpu(md[md_idx].len);
3236                 }
3237                 if (server_len == 0) {
3238                         if (!sgp) {
3239                                 dev_err(&vscsi->dev, "rdma: ran out of scatter/gather list\n");
3240                                 rc = -EIO;
3241                                 break;
3242                         }
3243                         server_ioba = sg_dma_address(sgp);
3244                         server_len = sg_dma_len(sgp);
3245                 }
3246
3247                 buf_len = tx_len;
3248
3249                 if (buf_len > client_len)
3250                         buf_len = client_len;
3251
3252                 if (buf_len > server_len)
3253                         buf_len = server_len;
3254
3255                 if (buf_len > max_vdma_size)
3256                         buf_len = max_vdma_size;
3257
3258                 if (dir == DMA_TO_DEVICE) {
3259                         /* read from client */
3260                         rc = h_copy_rdma(buf_len,
3261                                          vscsi->dds.window[REMOTE].liobn,
3262                                          client_ioba,
3263                                          vscsi->dds.window[LOCAL].liobn,
3264                                          server_ioba);
3265                 } else {
3266                         /* The h_copy_rdma will cause phyp, running in another
3267                          * partition, to read memory, so we need to make sure
3268                          * the data has been written out, hence these syncs.
3269                          */
3270                         /* ensure that everything is in memory */
3271                         isync();
3272                         /* ensure that memory has been made visible */
3273                         dma_wmb();
3274                         rc = h_copy_rdma(buf_len,
3275                                          vscsi->dds.window[LOCAL].liobn,
3276                                          server_ioba,
3277                                          vscsi->dds.window[REMOTE].liobn,
3278                                          client_ioba);
3279                 }
3280                 switch (rc) {
3281                 case H_SUCCESS:
3282                         break;
3283                 case H_PERMISSION:
3284                 case H_SOURCE_PARM:
3285                 case H_DEST_PARM:
3286                         if (connection_broken(vscsi)) {
3287                                 spin_lock_bh(&vscsi->intr_lock);
3288                                 vscsi->flags |=
3289                                         (RESPONSE_Q_DOWN | CLIENT_FAILED);
3290                                 spin_unlock_bh(&vscsi->intr_lock);
3291                         }
3292                         dev_err(&vscsi->dev, "rdma: h_copy_rdma failed, rc %ld\n",
3293                                 rc);
3294                         break;
3295
3296                 default:
3297                         dev_err(&vscsi->dev, "rdma: unknown error %ld from h_copy_rdma\n",
3298                                 rc);
3299                         break;
3300                 }
3301
3302                 if (!rc) {
3303                         tx_len -= buf_len;
3304                         if (tx_len) {
3305                                 client_len -= buf_len;
3306                                 if (client_len == 0)
3307                                         md_idx++;
3308                                 else
3309                                         client_ioba += buf_len;
3310
3311                                 server_len -= buf_len;
3312                                 if (server_len == 0)
3313                                         sgp = sg_next(sgp);
3314                                 else
3315                                         server_ioba += buf_len;
3316                         } else {
3317                                 break;
3318                         }
3319                 }
3320         } while (!rc);
3321
3322         return rc;
3323 }
3324
3325 /**
3326  * ibmvscsis_handle_crq() - Handle CRQ
3327  * @data:       Pointer to our adapter structure
3328  *
3329  * Read the command elements from the command queue and copy the payloads
3330  * associated with the command elements to local memory and execute the
3331  * SRP requests.
3332  *
3333  * Note: this is an edge triggered interrupt. It can not be shared.
3334  */
3335 static void ibmvscsis_handle_crq(unsigned long data)
3336 {
3337         struct scsi_info *vscsi = (struct scsi_info *)data;
3338         struct viosrp_crq *crq;
3339         long rc;
3340         bool ack = true;
3341         volatile u8 valid;
3342
3343         spin_lock_bh(&vscsi->intr_lock);
3344
3345         dev_dbg(&vscsi->dev, "got interrupt\n");
3346
3347         /*
3348          * if we are in a path where we are waiting for all pending commands
3349          * to complete because we received a transport event and anything in
3350          * the command queue is for a new connection, do nothing
3351          */
3352         if (TARGET_STOP(vscsi)) {
3353                 vio_enable_interrupts(vscsi->dma_dev);
3354
3355                 dev_dbg(&vscsi->dev, "handle_crq, don't process: flags 0x%x, state 0x%hx\n",
3356                         vscsi->flags, vscsi->state);
3357                 spin_unlock_bh(&vscsi->intr_lock);
3358                 return;
3359         }
3360
3361         rc = vscsi->flags & SCHEDULE_DISCONNECT;
3362         crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
3363         valid = crq->valid;
3364         dma_rmb();
3365
3366         while (valid) {
3367                 /*
3368                  * These are edege triggered interrupts. After dropping out of
3369                  * the while loop, the code must check for work since an
3370                  * interrupt could be lost, and an elment be left on the queue,
3371                  * hence the label.
3372                  */
3373 cmd_work:
3374                 vscsi->cmd_q.index =
3375                         (vscsi->cmd_q.index + 1) & vscsi->cmd_q.mask;
3376
3377                 if (!rc) {
3378                         rc = ibmvscsis_parse_command(vscsi, crq);
3379                 } else {
3380                         if ((uint)crq->valid == VALID_TRANS_EVENT) {
3381                                 /*
3382                                  * must service the transport layer events even
3383                                  * in an error state, dont break out until all
3384                                  * the consecutive transport events have been
3385                                  * processed
3386                                  */
3387                                 rc = ibmvscsis_trans_event(vscsi, crq);
3388                         } else if (vscsi->flags & TRANS_EVENT) {
3389                                 /*
3390                                  * if a transport event has occurred leave
3391                                  * everything but transport events on the queue
3392                                  *
3393                                  * need to decrement the queue index so we can
3394                                  * look at the element again
3395                                  */
3396                                 if (vscsi->cmd_q.index)
3397                                         vscsi->cmd_q.index -= 1;
3398                                 else
3399                                         /*
3400                                          * index is at 0 it just wrapped.
3401                                          * have it index last element in q
3402                                          */
3403                                         vscsi->cmd_q.index = vscsi->cmd_q.mask;
3404                                 break;
3405                         }
3406                 }
3407
3408                 crq->valid = INVALIDATE_CMD_RESP_EL;
3409
3410                 crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
3411                 valid = crq->valid;
3412                 dma_rmb();
3413         }
3414
3415         if (!rc) {
3416                 if (ack) {
3417                         vio_enable_interrupts(vscsi->dma_dev);
3418                         ack = false;
3419                         dev_dbg(&vscsi->dev, "handle_crq, reenabling interrupts\n");
3420                 }
3421                 valid = crq->valid;
3422                 dma_rmb();
3423                 if (valid)
3424                         goto cmd_work;
3425         } else {
3426                 dev_dbg(&vscsi->dev, "handle_crq, error: flags 0x%x, state 0x%hx, crq index 0x%x\n",
3427                         vscsi->flags, vscsi->state, vscsi->cmd_q.index);
3428         }
3429
3430         dev_dbg(&vscsi->dev, "Leaving handle_crq: schedule_q empty %d, flags 0x%x, state 0x%hx\n",
3431                 (int)list_empty(&vscsi->schedule_q), vscsi->flags,
3432                 vscsi->state);
3433
3434         spin_unlock_bh(&vscsi->intr_lock);
3435 }
3436
3437 static int ibmvscsis_probe(struct vio_dev *vdev,
3438                            const struct vio_device_id *id)
3439 {
3440         struct scsi_info *vscsi;
3441         int rc = 0;
3442         long hrc = 0;
3443         char wq_name[24];
3444
3445         vscsi = kzalloc(sizeof(*vscsi), GFP_KERNEL);
3446         if (!vscsi) {
3447                 rc = -ENOMEM;
3448                 dev_err(&vdev->dev, "probe: allocation of adapter failed\n");
3449                 return rc;
3450         }
3451
3452         vscsi->dma_dev = vdev;
3453         vscsi->dev = vdev->dev;
3454         INIT_LIST_HEAD(&vscsi->schedule_q);
3455         INIT_LIST_HEAD(&vscsi->waiting_rsp);
3456         INIT_LIST_HEAD(&vscsi->active_q);
3457
3458         snprintf(vscsi->tport.tport_name, IBMVSCSIS_NAMELEN, "%s",
3459                  dev_name(&vdev->dev));
3460
3461         dev_dbg(&vscsi->dev, "probe tport_name: %s\n", vscsi->tport.tport_name);
3462
3463         rc = read_dma_window(vscsi);
3464         if (rc)
3465                 goto free_adapter;
3466         dev_dbg(&vscsi->dev, "Probe: liobn 0x%x, riobn 0x%x\n",
3467                 vscsi->dds.window[LOCAL].liobn,
3468                 vscsi->dds.window[REMOTE].liobn);
3469
3470         snprintf(vscsi->eye, sizeof(vscsi->eye), "VSCSI %s", vdev->name);
3471
3472         vscsi->dds.unit_id = vdev->unit_address;
3473         strscpy(vscsi->dds.partition_name, partition_name,
3474                 sizeof(vscsi->dds.partition_name));
3475         vscsi->dds.partition_num = partition_number;
3476
3477         spin_lock_bh(&ibmvscsis_dev_lock);
3478         list_add_tail(&vscsi->list, &ibmvscsis_dev_list);
3479         spin_unlock_bh(&ibmvscsis_dev_lock);
3480
3481         /*
3482          * TBD: How do we determine # of cmds to request?  Do we know how
3483          * many "children" we have?
3484          */
3485         vscsi->request_limit = INITIAL_SRP_LIMIT;
3486         rc = srp_target_alloc(&vscsi->target, &vdev->dev, vscsi->request_limit,
3487                               SRP_MAX_IU_LEN);
3488         if (rc)
3489                 goto rem_list;
3490
3491         vscsi->target.ldata = vscsi;
3492
3493         rc = ibmvscsis_alloc_cmds(vscsi, vscsi->request_limit);
3494         if (rc) {
3495                 dev_err(&vscsi->dev, "alloc_cmds failed, rc %d, num %d\n",
3496                         rc, vscsi->request_limit);
3497                 goto free_target;
3498         }
3499
3500         /*
3501          * Note: the lock is used in freeing timers, so must initialize
3502          * first so that ordering in case of error is correct.
3503          */
3504         spin_lock_init(&vscsi->intr_lock);
3505
3506         rc = ibmvscsis_alloctimer(vscsi);
3507         if (rc) {
3508                 dev_err(&vscsi->dev, "probe: alloctimer failed, rc %d\n", rc);
3509                 goto free_cmds;
3510         }
3511
3512         rc = ibmvscsis_create_command_q(vscsi, 256);
3513         if (rc) {
3514                 dev_err(&vscsi->dev, "probe: create_command_q failed, rc %d\n",
3515                         rc);
3516                 goto free_timer;
3517         }
3518
3519         vscsi->map_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
3520         if (!vscsi->map_buf) {
3521                 rc = -ENOMEM;
3522                 dev_err(&vscsi->dev, "probe: allocating cmd buffer failed\n");
3523                 goto destroy_queue;
3524         }
3525
3526         vscsi->map_ioba = dma_map_single(&vdev->dev, vscsi->map_buf, PAGE_SIZE,
3527                                          DMA_BIDIRECTIONAL);
3528         if (dma_mapping_error(&vdev->dev, vscsi->map_ioba)) {
3529                 rc = -ENOMEM;
3530                 dev_err(&vscsi->dev, "probe: error mapping command buffer\n");
3531                 goto free_buf;
3532         }
3533
3534         hrc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
3535                        (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
3536                        0);
3537         if (hrc == H_SUCCESS)
3538                 vscsi->client_data.partition_number =
3539                         be64_to_cpu(*(u64 *)vscsi->map_buf);
3540         /*
3541          * We expect the VIOCTL to fail if we're configured as "any
3542          * client can connect" and the client isn't activated yet.
3543          * We'll make the call again when he sends an init msg.
3544          */
3545         dev_dbg(&vscsi->dev, "probe hrc %ld, client partition num %d\n",
3546                 hrc, vscsi->client_data.partition_number);
3547
3548         tasklet_init(&vscsi->work_task, ibmvscsis_handle_crq,
3549                      (unsigned long)vscsi);
3550
3551         init_completion(&vscsi->wait_idle);
3552         init_completion(&vscsi->unconfig);
3553
3554         snprintf(wq_name, 24, "ibmvscsis%s", dev_name(&vdev->dev));
3555         vscsi->work_q = create_workqueue(wq_name);
3556         if (!vscsi->work_q) {
3557                 rc = -ENOMEM;
3558                 dev_err(&vscsi->dev, "create_workqueue failed\n");
3559                 goto unmap_buf;
3560         }
3561
3562         rc = request_irq(vdev->irq, ibmvscsis_interrupt, 0, "ibmvscsis", vscsi);
3563         if (rc) {
3564                 rc = -EPERM;
3565                 dev_err(&vscsi->dev, "probe: request_irq failed, rc %d\n", rc);
3566                 goto destroy_WQ;
3567         }
3568
3569         vscsi->state = WAIT_ENABLED;
3570
3571         dev_set_drvdata(&vdev->dev, vscsi);
3572
3573         return 0;
3574
3575 destroy_WQ:
3576         destroy_workqueue(vscsi->work_q);
3577 unmap_buf:
3578         dma_unmap_single(&vdev->dev, vscsi->map_ioba, PAGE_SIZE,
3579                          DMA_BIDIRECTIONAL);
3580 free_buf:
3581         kfree(vscsi->map_buf);
3582 destroy_queue:
3583         tasklet_kill(&vscsi->work_task);
3584         ibmvscsis_unregister_command_q(vscsi);
3585         ibmvscsis_destroy_command_q(vscsi);
3586 free_timer:
3587         ibmvscsis_freetimer(vscsi);
3588 free_cmds:
3589         ibmvscsis_free_cmds(vscsi);
3590 free_target:
3591         srp_target_free(&vscsi->target);
3592 rem_list:
3593         spin_lock_bh(&ibmvscsis_dev_lock);
3594         list_del(&vscsi->list);
3595         spin_unlock_bh(&ibmvscsis_dev_lock);
3596 free_adapter:
3597         kfree(vscsi);
3598
3599         return rc;
3600 }
3601
3602 static int ibmvscsis_remove(struct vio_dev *vdev)
3603 {
3604         struct scsi_info *vscsi = dev_get_drvdata(&vdev->dev);
3605
3606         dev_dbg(&vscsi->dev, "remove (%s)\n", dev_name(&vscsi->dma_dev->dev));
3607
3608         spin_lock_bh(&vscsi->intr_lock);
3609         ibmvscsis_post_disconnect(vscsi, UNCONFIGURING, 0);
3610         vscsi->flags |= CFG_SLEEPING;
3611         spin_unlock_bh(&vscsi->intr_lock);
3612         wait_for_completion(&vscsi->unconfig);
3613
3614         vio_disable_interrupts(vdev);
3615         free_irq(vdev->irq, vscsi);
3616         destroy_workqueue(vscsi->work_q);
3617         dma_unmap_single(&vdev->dev, vscsi->map_ioba, PAGE_SIZE,
3618                          DMA_BIDIRECTIONAL);
3619         kfree(vscsi->map_buf);
3620         tasklet_kill(&vscsi->work_task);
3621         ibmvscsis_destroy_command_q(vscsi);
3622         ibmvscsis_freetimer(vscsi);
3623         ibmvscsis_free_cmds(vscsi);
3624         srp_target_free(&vscsi->target);
3625         spin_lock_bh(&ibmvscsis_dev_lock);
3626         list_del(&vscsi->list);
3627         spin_unlock_bh(&ibmvscsis_dev_lock);
3628         kfree(vscsi);
3629
3630         return 0;
3631 }
3632
3633 static ssize_t system_id_show(struct device *dev,
3634                               struct device_attribute *attr, char *buf)
3635 {
3636         return snprintf(buf, PAGE_SIZE, "%s\n", system_id);
3637 }
3638
3639 static ssize_t partition_number_show(struct device *dev,
3640                                      struct device_attribute *attr, char *buf)
3641 {
3642         return snprintf(buf, PAGE_SIZE, "%x\n", partition_number);
3643 }
3644
3645 static ssize_t unit_address_show(struct device *dev,
3646                                  struct device_attribute *attr, char *buf)
3647 {
3648         struct scsi_info *vscsi = container_of(dev, struct scsi_info, dev);
3649
3650         return snprintf(buf, PAGE_SIZE, "%x\n", vscsi->dma_dev->unit_address);
3651 }
3652
3653 static int ibmvscsis_get_system_info(void)
3654 {
3655         struct device_node *rootdn, *vdevdn;
3656         const char *id, *model, *name;
3657         const uint *num;
3658
3659         rootdn = of_find_node_by_path("/");
3660         if (!rootdn)
3661                 return -ENOENT;
3662
3663         model = of_get_property(rootdn, "model", NULL);
3664         id = of_get_property(rootdn, "system-id", NULL);
3665         if (model && id)
3666                 snprintf(system_id, sizeof(system_id), "%s-%s", model, id);
3667
3668         name = of_get_property(rootdn, "ibm,partition-name", NULL);
3669         if (name)
3670                 strncpy(partition_name, name, sizeof(partition_name));
3671
3672         num = of_get_property(rootdn, "ibm,partition-no", NULL);
3673         if (num)
3674                 partition_number = of_read_number(num, 1);
3675
3676         of_node_put(rootdn);
3677
3678         vdevdn = of_find_node_by_path("/vdevice");
3679         if (vdevdn) {
3680                 const uint *mvds;
3681
3682                 mvds = of_get_property(vdevdn, "ibm,max-virtual-dma-size",
3683                                        NULL);
3684                 if (mvds)
3685                         max_vdma_size = *mvds;
3686                 of_node_put(vdevdn);
3687         }
3688
3689         return 0;
3690 }
3691
3692 static char *ibmvscsis_get_fabric_wwn(struct se_portal_group *se_tpg)
3693 {
3694         struct ibmvscsis_tport *tport =
3695                 container_of(se_tpg, struct ibmvscsis_tport, se_tpg);
3696
3697         return tport->tport_name;
3698 }
3699
3700 static u16 ibmvscsis_get_tag(struct se_portal_group *se_tpg)
3701 {
3702         struct ibmvscsis_tport *tport =
3703                 container_of(se_tpg, struct ibmvscsis_tport, se_tpg);
3704
3705         return tport->tport_tpgt;
3706 }
3707
3708 static u32 ibmvscsis_get_default_depth(struct se_portal_group *se_tpg)
3709 {
3710         return 1;
3711 }
3712
3713 static int ibmvscsis_check_true(struct se_portal_group *se_tpg)
3714 {
3715         return 1;
3716 }
3717
3718 static int ibmvscsis_check_false(struct se_portal_group *se_tpg)
3719 {
3720         return 0;
3721 }
3722
3723 static u32 ibmvscsis_tpg_get_inst_index(struct se_portal_group *se_tpg)
3724 {
3725         return 1;
3726 }
3727
3728 static int ibmvscsis_check_stop_free(struct se_cmd *se_cmd)
3729 {
3730         return target_put_sess_cmd(se_cmd);
3731 }
3732
3733 static void ibmvscsis_release_cmd(struct se_cmd *se_cmd)
3734 {
3735         struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3736                                                  se_cmd);
3737         struct scsi_info *vscsi = cmd->adapter;
3738
3739         spin_lock_bh(&vscsi->intr_lock);
3740         /* Remove from active_q */
3741         list_move_tail(&cmd->list, &vscsi->waiting_rsp);
3742         ibmvscsis_send_messages(vscsi);
3743         spin_unlock_bh(&vscsi->intr_lock);
3744 }
3745
3746 static u32 ibmvscsis_sess_get_index(struct se_session *se_sess)
3747 {
3748         return 0;
3749 }
3750
3751 static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
3752 {
3753         struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3754                                                  se_cmd);
3755         struct scsi_info *vscsi = cmd->adapter;
3756         struct iu_entry *iue = cmd->iue;
3757         int rc;
3758
3759         /*
3760          * If CLIENT_FAILED OR RESPONSE_Q_DOWN, then just return success
3761          * since LIO can't do anything about it, and we dont want to
3762          * attempt an srp_transfer_data.
3763          */
3764         if ((vscsi->flags & (CLIENT_FAILED | RESPONSE_Q_DOWN))) {
3765                 dev_err(&vscsi->dev, "write_pending failed since: %d\n",
3766                         vscsi->flags);
3767                 return -EIO;
3768
3769         }
3770
3771         rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma,
3772                                1, 1);
3773         if (rc) {
3774                 dev_err(&vscsi->dev, "srp_transfer_data() failed: %d\n", rc);
3775                 return -EIO;
3776         }
3777         /*
3778          * We now tell TCM to add this WRITE CDB directly into the TCM storage
3779          * object execution queue.
3780          */
3781         target_execute_cmd(se_cmd);
3782         return 0;
3783 }
3784
3785 static void ibmvscsis_set_default_node_attrs(struct se_node_acl *nacl)
3786 {
3787 }
3788
3789 static int ibmvscsis_get_cmd_state(struct se_cmd *se_cmd)
3790 {
3791         return 0;
3792 }
3793
3794 static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd)
3795 {
3796         struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3797                                                  se_cmd);
3798         struct iu_entry *iue = cmd->iue;
3799         struct scsi_info *vscsi = cmd->adapter;
3800         char *sd;
3801         uint len = 0;
3802         int rc;
3803
3804         rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma, 1,
3805                                1);
3806         if (rc) {
3807                 dev_err(&vscsi->dev, "srp_transfer_data failed: %d\n", rc);
3808                 sd = se_cmd->sense_buffer;
3809                 se_cmd->scsi_sense_length = 18;
3810                 memset(se_cmd->sense_buffer, 0, se_cmd->scsi_sense_length);
3811                 /* Logical Unit Communication Time-out asc/ascq = 0x0801 */
3812                 scsi_build_sense_buffer(0, se_cmd->sense_buffer, MEDIUM_ERROR,
3813                                         0x08, 0x01);
3814         }
3815
3816         srp_build_response(vscsi, cmd, &len);
3817         cmd->rsp.format = SRP_FORMAT;
3818         cmd->rsp.len = len;
3819
3820         return 0;
3821 }
3822
3823 static int ibmvscsis_queue_status(struct se_cmd *se_cmd)
3824 {
3825         struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3826                                                  se_cmd);
3827         struct scsi_info *vscsi = cmd->adapter;
3828         uint len;
3829
3830         dev_dbg(&vscsi->dev, "queue_status %p\n", se_cmd);
3831
3832         srp_build_response(vscsi, cmd, &len);
3833         cmd->rsp.format = SRP_FORMAT;
3834         cmd->rsp.len = len;
3835
3836         return 0;
3837 }
3838
3839 static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd)
3840 {
3841         struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3842                                                  se_cmd);
3843         struct scsi_info *vscsi = cmd->adapter;
3844         struct ibmvscsis_cmd *cmd_itr;
3845         struct iu_entry *iue = iue = cmd->iue;
3846         struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
3847         u64 tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
3848         uint len;
3849
3850         dev_dbg(&vscsi->dev, "queue_tm_rsp %p, status %d\n",
3851                 se_cmd, (int)se_cmd->se_tmr_req->response);
3852
3853         if (srp_tsk->tsk_mgmt_func == SRP_TSK_ABORT_TASK &&
3854             cmd->se_cmd.se_tmr_req->response == TMR_TASK_DOES_NOT_EXIST) {
3855                 spin_lock_bh(&vscsi->intr_lock);
3856                 list_for_each_entry(cmd_itr, &vscsi->active_q, list) {
3857                         if (tag_to_abort == cmd_itr->se_cmd.tag) {
3858                                 cmd_itr->abort_cmd = cmd;
3859                                 cmd->flags |= DELAY_SEND;
3860                                 break;
3861                         }
3862                 }
3863                 spin_unlock_bh(&vscsi->intr_lock);
3864         }
3865
3866         srp_build_response(vscsi, cmd, &len);
3867         cmd->rsp.format = SRP_FORMAT;
3868         cmd->rsp.len = len;
3869 }
3870
3871 static void ibmvscsis_aborted_task(struct se_cmd *se_cmd)
3872 {
3873         struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3874                                                  se_cmd);
3875         struct scsi_info *vscsi = cmd->adapter;
3876
3877         dev_dbg(&vscsi->dev, "ibmvscsis_aborted_task %p task_tag: %llu\n",
3878                 se_cmd, se_cmd->tag);
3879 }
3880
3881 static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf,
3882                                            struct config_group *group,
3883                                            const char *name)
3884 {
3885         struct ibmvscsis_tport *tport;
3886         struct scsi_info *vscsi;
3887
3888         tport = ibmvscsis_lookup_port(name);
3889         if (tport) {
3890                 vscsi = container_of(tport, struct scsi_info, tport);
3891                 tport->tport_proto_id = SCSI_PROTOCOL_SRP;
3892                 dev_dbg(&vscsi->dev, "make_tport(%s), pointer:%p, tport_id:%x\n",
3893                         name, tport, tport->tport_proto_id);
3894                 return &tport->tport_wwn;
3895         }
3896
3897         return ERR_PTR(-EINVAL);
3898 }
3899
3900 static void ibmvscsis_drop_tport(struct se_wwn *wwn)
3901 {
3902         struct ibmvscsis_tport *tport = container_of(wwn,
3903                                                      struct ibmvscsis_tport,
3904                                                      tport_wwn);
3905         struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
3906
3907         dev_dbg(&vscsi->dev, "drop_tport(%s)\n",
3908                 config_item_name(&tport->tport_wwn.wwn_group.cg_item));
3909 }
3910
3911 static struct se_portal_group *ibmvscsis_make_tpg(struct se_wwn *wwn,
3912                                                   const char *name)
3913 {
3914         struct ibmvscsis_tport *tport =
3915                 container_of(wwn, struct ibmvscsis_tport, tport_wwn);
3916         u16 tpgt;
3917         int rc;
3918
3919         if (strstr(name, "tpgt_") != name)
3920                 return ERR_PTR(-EINVAL);
3921         rc = kstrtou16(name + 5, 0, &tpgt);
3922         if (rc)
3923                 return ERR_PTR(rc);
3924         tport->tport_tpgt = tpgt;
3925
3926         tport->releasing = false;
3927
3928         rc = core_tpg_register(&tport->tport_wwn, &tport->se_tpg,
3929                                tport->tport_proto_id);
3930         if (rc)
3931                 return ERR_PTR(rc);
3932
3933         return &tport->se_tpg;
3934 }
3935
3936 static void ibmvscsis_drop_tpg(struct se_portal_group *se_tpg)
3937 {
3938         struct ibmvscsis_tport *tport = container_of(se_tpg,
3939                                                      struct ibmvscsis_tport,
3940                                                      se_tpg);
3941
3942         tport->releasing = true;
3943         tport->enabled = false;
3944
3945         /*
3946          * Release the virtual I_T Nexus for this ibmvscsis TPG
3947          */
3948         ibmvscsis_drop_nexus(tport);
3949         /*
3950          * Deregister the se_tpg from TCM..
3951          */
3952         core_tpg_deregister(se_tpg);
3953 }
3954
3955 static ssize_t ibmvscsis_wwn_version_show(struct config_item *item,
3956                                           char *page)
3957 {
3958         return scnprintf(page, PAGE_SIZE, "%s\n", IBMVSCSIS_VERSION);
3959 }
3960 CONFIGFS_ATTR_RO(ibmvscsis_wwn_, version);
3961
3962 static struct configfs_attribute *ibmvscsis_wwn_attrs[] = {
3963         &ibmvscsis_wwn_attr_version,
3964         NULL,
3965 };
3966
3967 static ssize_t ibmvscsis_tpg_enable_show(struct config_item *item,
3968                                          char *page)
3969 {
3970         struct se_portal_group *se_tpg = to_tpg(item);
3971         struct ibmvscsis_tport *tport = container_of(se_tpg,
3972                                                      struct ibmvscsis_tport,
3973                                                      se_tpg);
3974
3975         return snprintf(page, PAGE_SIZE, "%d\n", (tport->enabled) ? 1 : 0);
3976 }
3977
3978 static ssize_t ibmvscsis_tpg_enable_store(struct config_item *item,
3979                                           const char *page, size_t count)
3980 {
3981         struct se_portal_group *se_tpg = to_tpg(item);
3982         struct ibmvscsis_tport *tport = container_of(se_tpg,
3983                                                      struct ibmvscsis_tport,
3984                                                      se_tpg);
3985         struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
3986         unsigned long tmp;
3987         int rc;
3988         long lrc;
3989
3990         rc = kstrtoul(page, 0, &tmp);
3991         if (rc < 0) {
3992                 dev_err(&vscsi->dev, "Unable to extract srpt_tpg_store_enable\n");
3993                 return -EINVAL;
3994         }
3995
3996         if ((tmp != 0) && (tmp != 1)) {
3997                 dev_err(&vscsi->dev, "Illegal value for srpt_tpg_store_enable\n");
3998                 return -EINVAL;
3999         }
4000
4001         if (tmp) {
4002                 spin_lock_bh(&vscsi->intr_lock);
4003                 tport->enabled = true;
4004                 lrc = ibmvscsis_enable_change_state(vscsi);
4005                 if (lrc)
4006                         dev_err(&vscsi->dev, "enable_change_state failed, rc %ld state %d\n",
4007                                 lrc, vscsi->state);
4008                 spin_unlock_bh(&vscsi->intr_lock);
4009         } else {
4010                 spin_lock_bh(&vscsi->intr_lock);
4011                 tport->enabled = false;
4012                 /* This simulates the server going down */
4013                 ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
4014                 spin_unlock_bh(&vscsi->intr_lock);
4015         }
4016
4017         dev_dbg(&vscsi->dev, "tpg_enable_store, tmp %ld, state %d\n", tmp,
4018                 vscsi->state);
4019
4020         return count;
4021 }
4022 CONFIGFS_ATTR(ibmvscsis_tpg_, enable);
4023
4024 static struct configfs_attribute *ibmvscsis_tpg_attrs[] = {
4025         &ibmvscsis_tpg_attr_enable,
4026         NULL,
4027 };
4028
4029 static const struct target_core_fabric_ops ibmvscsis_ops = {
4030         .module                         = THIS_MODULE,
4031         .fabric_name                    = "ibmvscsis",
4032         .max_data_sg_nents              = MAX_TXU / PAGE_SIZE,
4033         .tpg_get_wwn                    = ibmvscsis_get_fabric_wwn,
4034         .tpg_get_tag                    = ibmvscsis_get_tag,
4035         .tpg_get_default_depth          = ibmvscsis_get_default_depth,
4036         .tpg_check_demo_mode            = ibmvscsis_check_true,
4037         .tpg_check_demo_mode_cache      = ibmvscsis_check_true,
4038         .tpg_check_demo_mode_write_protect = ibmvscsis_check_false,
4039         .tpg_check_prod_mode_write_protect = ibmvscsis_check_false,
4040         .tpg_get_inst_index             = ibmvscsis_tpg_get_inst_index,
4041         .check_stop_free                = ibmvscsis_check_stop_free,
4042         .release_cmd                    = ibmvscsis_release_cmd,
4043         .sess_get_index                 = ibmvscsis_sess_get_index,
4044         .write_pending                  = ibmvscsis_write_pending,
4045         .set_default_node_attributes    = ibmvscsis_set_default_node_attrs,
4046         .get_cmd_state                  = ibmvscsis_get_cmd_state,
4047         .queue_data_in                  = ibmvscsis_queue_data_in,
4048         .queue_status                   = ibmvscsis_queue_status,
4049         .queue_tm_rsp                   = ibmvscsis_queue_tm_rsp,
4050         .aborted_task                   = ibmvscsis_aborted_task,
4051         /*
4052          * Setup function pointers for logic in target_core_fabric_configfs.c
4053          */
4054         .fabric_make_wwn                = ibmvscsis_make_tport,
4055         .fabric_drop_wwn                = ibmvscsis_drop_tport,
4056         .fabric_make_tpg                = ibmvscsis_make_tpg,
4057         .fabric_drop_tpg                = ibmvscsis_drop_tpg,
4058
4059         .tfc_wwn_attrs                  = ibmvscsis_wwn_attrs,
4060         .tfc_tpg_base_attrs             = ibmvscsis_tpg_attrs,
4061 };
4062
4063 static void ibmvscsis_dev_release(struct device *dev) {};
4064
4065 static struct device_attribute dev_attr_system_id =
4066         __ATTR(system_id, S_IRUGO, system_id_show, NULL);
4067
4068 static struct device_attribute dev_attr_partition_number =
4069         __ATTR(partition_number, S_IRUGO, partition_number_show, NULL);
4070
4071 static struct device_attribute dev_attr_unit_address =
4072         __ATTR(unit_address, S_IRUGO, unit_address_show, NULL);
4073
4074 static struct attribute *ibmvscsis_dev_attrs[] = {
4075         &dev_attr_system_id.attr,
4076         &dev_attr_partition_number.attr,
4077         &dev_attr_unit_address.attr,
4078 };
4079 ATTRIBUTE_GROUPS(ibmvscsis_dev);
4080
4081 static struct class ibmvscsis_class = {
4082         .name           = "ibmvscsis",
4083         .dev_release    = ibmvscsis_dev_release,
4084         .dev_groups     = ibmvscsis_dev_groups,
4085 };
4086
4087 static const struct vio_device_id ibmvscsis_device_table[] = {
4088         { "v-scsi-host", "IBM,v-scsi-host" },
4089         { "", "" }
4090 };
4091 MODULE_DEVICE_TABLE(vio, ibmvscsis_device_table);
4092
4093 static struct vio_driver ibmvscsis_driver = {
4094         .name = "ibmvscsis",
4095         .id_table = ibmvscsis_device_table,
4096         .probe = ibmvscsis_probe,
4097         .remove = ibmvscsis_remove,
4098 };
4099
4100 /*
4101  * ibmvscsis_init() - Kernel Module initialization
4102  *
4103  * Note: vio_register_driver() registers callback functions, and at least one
4104  * of those callback functions calls TCM - Linux IO Target Subsystem, thus
4105  * the SCSI Target template must be registered before vio_register_driver()
4106  * is called.
4107  */
4108 static int __init ibmvscsis_init(void)
4109 {
4110         int rc = 0;
4111
4112         rc = ibmvscsis_get_system_info();
4113         if (rc) {
4114                 pr_err("rc %d from get_system_info\n", rc);
4115                 goto out;
4116         }
4117
4118         rc = class_register(&ibmvscsis_class);
4119         if (rc) {
4120                 pr_err("failed class register\n");
4121                 goto out;
4122         }
4123
4124         rc = target_register_template(&ibmvscsis_ops);
4125         if (rc) {
4126                 pr_err("rc %d from target_register_template\n", rc);
4127                 goto unregister_class;
4128         }
4129
4130         rc = vio_register_driver(&ibmvscsis_driver);
4131         if (rc) {
4132                 pr_err("rc %d from vio_register_driver\n", rc);
4133                 goto unregister_target;
4134         }
4135
4136         return 0;
4137
4138 unregister_target:
4139         target_unregister_template(&ibmvscsis_ops);
4140 unregister_class:
4141         class_unregister(&ibmvscsis_class);
4142 out:
4143         return rc;
4144 }
4145
4146 static void __exit ibmvscsis_exit(void)
4147 {
4148         pr_info("Unregister IBM virtual SCSI host driver\n");
4149         vio_unregister_driver(&ibmvscsis_driver);
4150         target_unregister_template(&ibmvscsis_ops);
4151         class_unregister(&ibmvscsis_class);
4152 }
4153
4154 MODULE_DESCRIPTION("IBMVSCSIS fabric driver");
4155 MODULE_AUTHOR("Bryant G. Ly and Michael Cyr");
4156 MODULE_LICENSE("GPL");
4157 MODULE_VERSION(IBMVSCSIS_VERSION);
4158 module_init(ibmvscsis_init);
4159 module_exit(ibmvscsis_exit);