drivers/s390/cio/qdio_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Linux for s390 qdio support, buffer handling, qdio API and module support.
   4  *
   5  * Copyright IBM Corp. 2000, 2008
   6  * Author(s): Utz Bacher <[email protected]>
   7  *            Jan Glauber <[email protected]>
   8  * 2.6 cio integration by Cornelia Huck <[email protected]>
   9  */
  10 #include <linux/module.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/kmemleak.h>
  14 #include <linux/delay.h>
  15 #include <linux/gfp.h>
  16 #include <linux/io.h>
  17 #include <linux/atomic.h>
  18 #include <asm/debug.h>
  19 #include <asm/qdio.h>
  20 #include <asm/asm.h>
  21 #include <asm/ipl.h>
  22
  23 #include "cio.h"
  24 #include "css.h"
  25 #include "device.h"
  26 #include "qdio.h"
  27 #include "qdio_debug.h"
  28
  29 MODULE_AUTHOR("Utz Bacher <[email protected]>,"\
  30         "Jan Glauber <[email protected]>");
  31 MODULE_DESCRIPTION("QDIO base support");
  32 MODULE_LICENSE("GPL");
  33
  34 static inline int do_siga_sync(unsigned long schid,
  35                                unsigned long out_mask, unsigned long in_mask,
  36                                unsigned int fc)
  37 {
  38         int cc;
  39
  40         asm volatile(
  41                 "       lgr     0,%[fc]\n"
  42                 "       lgr     1,%[schid]\n"
  43                 "       lgr     2,%[out]\n"
  44                 "       lgr     3,%[in]\n"
  45                 "       siga    0\n"
  46                 CC_IPM(cc)
  47                 : CC_OUT(cc, cc)
  48                 : [fc] "d" (fc), [schid] "d" (schid),
  49                   [out] "d" (out_mask), [in] "d" (in_mask)
  50                 : CC_CLOBBER_LIST("0", "1", "2", "3"));
  51         return CC_TRANSFORM(cc);
  52 }
  53
  54 static inline int do_siga_input(unsigned long schid, unsigned long mask,
  55                                 unsigned long fc)
  56 {
  57         int cc;
  58
  59         asm volatile(
  60                 "       lgr     0,%[fc]\n"
  61                 "       lgr     1,%[schid]\n"
  62                 "       lgr     2,%[mask]\n"
  63                 "       siga    0\n"
  64                 CC_IPM(cc)
  65                 : CC_OUT(cc, cc)
  66                 : [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask)
  67                 : CC_CLOBBER_LIST("0", "1", "2"));
  68         return CC_TRANSFORM(cc);
  69 }
  70
  71 /**
  72  * do_siga_output - perform SIGA-w/wt function
  73  * @schid: subchannel id or in case of QEBSM the subchannel token
  74  * @mask: which output queues to process
  75  * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
  76  * @fc: function code to perform
  77  * @aob: asynchronous operation block
  78  *
  79  * Returns condition code.
  80  * Note: For IQDC unicast queues only the highest priority queue is processed.
  81  */
  82 static inline int do_siga_output(unsigned long schid, unsigned long mask,
  83                                  unsigned int *bb, unsigned long fc,
  84                                  dma64_t aob)
  85 {
  86         int cc;
  87
  88         asm volatile(
  89                 "       lgr     0,%[fc]\n"
  90                 "       lgr     1,%[schid]\n"
  91                 "       lgr     2,%[mask]\n"
  92                 "       lgr     3,%[aob]\n"
  93                 "       siga    0\n"
  94                 "       lgr     %[fc],0\n"
  95                 CC_IPM(cc)
  96                 : CC_OUT(cc, cc), [fc] "+&d" (fc)
  97                 : [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob)
  98                 : CC_CLOBBER_LIST("0", "1", "2", "3"));
  99         *bb = fc >> 31;
 100         return CC_TRANSFORM(cc);
 101 }
 102
 103 /**
 104  * qdio_do_eqbs - extract buffer states for QEBSM
 105  * @q: queue to manipulate
 106  * @state: state of the extracted buffers
 107  * @start: buffer number to start at
 108  * @count: count of buffers to examine
 109  * @auto_ack: automatically acknowledge buffers
 110  *
 111  * Returns the number of successfully extracted equal buffer states.
 112  * Stops processing if a state is different from the last buffers state.
 113  */
 114 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
 115                         int start, int count, int auto_ack)
 116 {
 117         int tmp_count = count, tmp_start = start, nr = q->nr;
 118         unsigned int ccq = 0;
 119
 120         qperf_inc(q, eqbs);
 121
 122         if (!q->is_input_q)
 123                 nr += q->irq_ptr->nr_input_qs;
 124 again:
 125         ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
 126                       auto_ack);
 127
 128         switch (ccq) {
 129         case 0:
 130         case 32:
 131                 /* all done, or next buffer state different */
 132                 return count - tmp_count;
 133         case 96:
 134                 /* not all buffers processed */
 135                 qperf_inc(q, eqbs_partial);
 136                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",
 137                         tmp_count);
 138                 return count - tmp_count;
 139         case 97:
 140                 /* no buffer processed */
 141                 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
 142                 goto again;
 143         default:
 144                 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
 145                 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
 146                 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 147                 q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
 148                            q->first_to_check, count, q->irq_ptr->int_parm);
 149                 return 0;
 150         }
 151 }
 152
 153 /**
 154  * qdio_do_sqbs - set buffer states for QEBSM
 155  * @q: queue to manipulate
 156  * @state: new state of the buffers
 157  * @start: first buffer number to change
 158  * @count: how many buffers to change
 159  *
 160  * Returns the number of successfully changed buffers.
 161  * Does retrying until the specified count of buffer states is set or an
 162  * error occurs.
 163  */
 164 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
 165                         int count)
 166 {
 167         unsigned int ccq = 0;
 168         int tmp_count = count, tmp_start = start;
 169         int nr = q->nr;
 170
 171         qperf_inc(q, sqbs);
 172
 173         if (!q->is_input_q)
 174                 nr += q->irq_ptr->nr_input_qs;
 175 again:
 176         ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
 177
 178         switch (ccq) {
 179         case 0:
 180         case 32:
 181                 /* all done, or active buffer adapter-owned */
 182                 WARN_ON_ONCE(tmp_count);
 183                 return count - tmp_count;
 184         case 96:
 185                 /* not all buffers processed */
 186                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
 187                 qperf_inc(q, sqbs_partial);
 188                 goto again;
 189         default:
 190                 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
 191                 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
 192                 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 193                 q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
 194                            q->first_to_check, count, q->irq_ptr->int_parm);
 195                 return 0;
 196         }
 197 }
 198
 199 /*
 200  * Returns number of examined buffers and their common state in *state.
 201  * Requested number of buffers-to-examine must be > 0.
 202  */
 203 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
 204                                  unsigned char *state, unsigned int count,
 205                                  int auto_ack)
 206 {
 207         unsigned char __state = 0;
 208         int i = 1;
 209
 210         if (is_qebsm(q))
 211                 return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
 212
 213         /* get initial state: */
 214         __state = q->slsb.val[bufnr];
 215
 216         /* Bail out early if there is no work on the queue: */
 217         if (__state & SLSB_OWNER_CU)
 218                 goto out;
 219
 220         for (; i < count; i++) {
 221                 bufnr = next_buf(bufnr);
 222
 223                 /* stop if next state differs from initial state: */
 224                 if (q->slsb.val[bufnr] != __state)
 225                         break;
 226         }
 227
 228 out:
 229         *state = __state;
 230         return i;
 231 }
 232
 233 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
 234                                 unsigned char *state, int auto_ack)
 235 {
 236         return get_buf_states(q, bufnr, state, 1, auto_ack);
 237 }
 238
 239 /* wrap-around safe setting of slsb states, returns number of changed buffers */
 240 static inline int set_buf_states(struct qdio_q *q, int bufnr,
 241                                  unsigned char state, int count)
 242 {
 243         int i;
 244
 245         if (is_qebsm(q))
 246                 return qdio_do_sqbs(q, state, bufnr, count);
 247
 248         /* Ensure that all preceding changes to the SBALs are visible: */
 249         mb();
 250
 251         for (i = 0; i < count; i++) {
 252                 WRITE_ONCE(q->slsb.val[bufnr], state);
 253                 bufnr = next_buf(bufnr);
 254         }
 255
 256         /* Make our SLSB changes visible: */
 257         mb();
 258
 259         return count;
 260 }
 261
 262 static inline int set_buf_state(struct qdio_q *q, int bufnr,
 263                                 unsigned char state)
 264 {
 265         return set_buf_states(q, bufnr, state, 1);
 266 }
 267
 268 /* set slsb states to initial state */
 269 static void qdio_init_buf_states(struct qdio_irq *irq_ptr)
 270 {
 271         struct qdio_q *q;
 272         int i;
 273
 274         for_each_input_queue(irq_ptr, q, i)
 275                 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
 276                                QDIO_MAX_BUFFERS_PER_Q);
 277         for_each_output_queue(irq_ptr, q, i)
 278                 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
 279                                QDIO_MAX_BUFFERS_PER_Q);
 280 }
 281
 282 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
 283                           unsigned int input)
 284 {
 285         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 286         unsigned int fc = QDIO_SIGA_SYNC;
 287         int cc;
 288
 289         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
 290         qperf_inc(q, siga_sync);
 291
 292         if (is_qebsm(q)) {
 293                 schid = q->irq_ptr->sch_token;
 294                 fc |= QDIO_SIGA_QEBSM_FLAG;
 295         }
 296
 297         cc = do_siga_sync(schid, output, input, fc);
 298         if (unlikely(cc))
 299                 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
 300         return (cc) ? -EIO : 0;
 301 }
 302
 303 static inline int qdio_sync_input_queue(struct qdio_q *q)
 304 {
 305         return qdio_siga_sync(q, 0, q->mask);
 306 }
 307
 308 static inline int qdio_sync_output_queue(struct qdio_q *q)
 309 {
 310         return qdio_siga_sync(q, q->mask, 0);
 311 }
 312
 313 static inline int qdio_siga_sync_q(struct qdio_q *q)
 314 {
 315         if (q->is_input_q)
 316                 return qdio_sync_input_queue(q);
 317         else
 318                 return qdio_sync_output_queue(q);
 319 }
 320
 321 static int qdio_siga_output(struct qdio_q *q, unsigned int count,
 322                             unsigned int *busy_bit, dma64_t aob)
 323 {
 324         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 325         unsigned int fc = QDIO_SIGA_WRITE;
 326         u64 start_time = 0;
 327         int retries = 0, cc;
 328
 329         if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {
 330                 if (count > 1)
 331                         fc = QDIO_SIGA_WRITEM;
 332                 else if (aob)
 333                         fc = QDIO_SIGA_WRITEQ;
 334         }
 335
 336         if (is_qebsm(q)) {
 337                 schid = q->irq_ptr->sch_token;
 338                 fc |= QDIO_SIGA_QEBSM_FLAG;
 339         }
 340 again:
 341         cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);
 342
 343         /* hipersocket busy condition */
 344         if (unlikely(*busy_bit)) {
 345                 retries++;
 346
 347                 if (!start_time) {
 348                         start_time = get_tod_clock_fast();
 349                         goto again;
 350                 }
 351                 if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE)
 352                         goto again;
 353         }
 354         if (retries) {
 355                 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,
 356                               "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);
 357                 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);
 358         }
 359         return cc;
 360 }
 361
 362 static inline int qdio_siga_input(struct qdio_q *q)
 363 {
 364         unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 365         unsigned int fc = QDIO_SIGA_READ;
 366         int cc;
 367
 368         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
 369         qperf_inc(q, siga_read);
 370
 371         if (is_qebsm(q)) {
 372                 schid = q->irq_ptr->sch_token;
 373                 fc |= QDIO_SIGA_QEBSM_FLAG;
 374         }
 375
 376         cc = do_siga_input(schid, q->mask, fc);
 377         if (unlikely(cc))
 378                 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
 379         return (cc) ? -EIO : 0;
 380 }
 381
 382 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
 383                         unsigned char *state)
 384 {
 385         if (qdio_need_siga_sync(q->irq_ptr))
 386                 qdio_siga_sync_q(q);
 387         return get_buf_state(q, bufnr, state, 0);
 388 }
 389
 390 static inline void qdio_stop_polling(struct qdio_q *q)
 391 {
 392         if (!q->u.in.batch_count)
 393                 return;
 394
 395         qperf_inc(q, stop_polling);
 396
 397         /* show the card that we are not polling anymore */
 398         set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,
 399                        q->u.in.batch_count);
 400         q->u.in.batch_count = 0;
 401 }
 402
 403 static inline void account_sbals(struct qdio_q *q, unsigned int count)
 404 {
 405         q->q_stats.nr_sbal_total += count;
 406         q->q_stats.nr_sbals[ilog2(count)]++;
 407 }
 408
 409 static void process_buffer_error(struct qdio_q *q, unsigned int start,
 410                                  int count)
 411 {
 412         /* special handling for no target buffer empty */
 413         if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&
 414             q->sbal[start]->element[15].sflags == 0x10) {
 415                 qperf_inc(q, target_full);
 416                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);
 417                 return;
 418         }
 419
 420         DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
 421         DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
 422         DBF_ERROR("FTC:%3d C:%3d", start, count);
 423         DBF_ERROR("F14:%2x F15:%2x",
 424                   q->sbal[start]->element[14].sflags,
 425                   q->sbal[start]->element[15].sflags);
 426 }
 427
 428 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
 429                                        int count, bool auto_ack)
 430 {
 431         /* ACK the newest SBAL: */
 432         if (!auto_ack)
 433                 set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);
 434
 435         if (!q->u.in.batch_count)
 436                 q->u.in.batch_start = start;
 437         q->u.in.batch_count += count;
 438 }
 439
 440 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start,
 441                                        unsigned int *error)
 442 {
 443         unsigned char state = 0;
 444         int count;
 445
 446         q->timestamp = get_tod_clock_fast();
 447
 448         count = atomic_read(&q->nr_buf_used);
 449         if (!count)
 450                 return 0;
 451
 452         if (qdio_need_siga_sync(q->irq_ptr))
 453                 qdio_sync_input_queue(q);
 454
 455         count = get_buf_states(q, start, &state, count, 1);
 456         if (!count)
 457                 return 0;
 458
 459         switch (state) {
 460         case SLSB_P_INPUT_PRIMED:
 461                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
 462                               count);
 463
 464                 inbound_handle_work(q, start, count, is_qebsm(q));
 465                 if (atomic_sub_return(count, &q->nr_buf_used) == 0)
 466                         qperf_inc(q, inbound_queue_full);
 467                 if (q->irq_ptr->perf_stat_enabled)
 468                         account_sbals(q, count);
 469                 return count;
 470         case SLSB_P_INPUT_ERROR:
 471                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
 472                               count);
 473
 474                 *error = QDIO_ERROR_SLSB_STATE;
 475                 process_buffer_error(q, start, count);
 476                 inbound_handle_work(q, start, count, false);
 477                 if (atomic_sub_return(count, &q->nr_buf_used) == 0)
 478                         qperf_inc(q, inbound_queue_full);
 479                 if (q->irq_ptr->perf_stat_enabled)
 480                         account_sbals_error(q, count);
 481                 return count;
 482         case SLSB_CU_INPUT_EMPTY:
 483                 if (q->irq_ptr->perf_stat_enabled)
 484                         q->q_stats.nr_sbal_nop++;
 485                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",
 486                               q->nr, start);
 487                 return 0;
 488         case SLSB_P_INPUT_NOT_INIT:
 489         case SLSB_P_INPUT_ACK:
 490                 /* We should never see this state, throw a WARN: */
 491         default:
 492                 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
 493                               "found state %#x at index %u on queue %u\n",
 494                               state, start, q->nr);
 495                 return 0;
 496         }
 497 }
 498
 499 int qdio_inspect_input_queue(struct ccw_device *cdev, unsigned int nr,
 500                              unsigned int *bufnr, unsigned int *error)
 501 {
 502         struct qdio_irq *irq = cdev->private->qdio_data;
 503         unsigned int start;
 504         struct qdio_q *q;
 505         int count;
 506
 507         if (!irq)
 508                 return -ENODEV;
 509
 510         q = irq->input_qs[nr];
 511         start = q->first_to_check;
 512         *error = 0;
 513
 514         count = get_inbound_buffer_frontier(q, start, error);
 515         if (count == 0)
 516                 return 0;
 517
 518         *bufnr = start;
 519         q->first_to_check = add_buf(start, count);
 520         return count;
 521 }
 522 EXPORT_SYMBOL_GPL(qdio_inspect_input_queue);
 523
 524 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)
 525 {
 526         unsigned char state = 0;
 527
 528         if (!atomic_read(&q->nr_buf_used))
 529                 return 1;
 530
 531         if (qdio_need_siga_sync(q->irq_ptr))
 532                 qdio_sync_input_queue(q);
 533         get_buf_state(q, start, &state, 0);
 534
 535         if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
 536                 /* more work coming */
 537                 return 0;
 538
 539         return 1;
 540 }
 541
 542 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start,
 543                                         unsigned int *error)
 544 {
 545         unsigned char state = 0;
 546         int count;
 547
 548         q->timestamp = get_tod_clock_fast();
 549
 550         count = atomic_read(&q->nr_buf_used);
 551         if (!count)
 552                 return 0;
 553
 554         if (qdio_need_siga_sync(q->irq_ptr))
 555                 qdio_sync_output_queue(q);
 556
 557         count = get_buf_states(q, start, &state, count, 0);
 558         if (!count)
 559                 return 0;
 560
 561         switch (state) {
 562         case SLSB_P_OUTPUT_PENDING:
 563                 *error = QDIO_ERROR_SLSB_PENDING;
 564                 fallthrough;
 565         case SLSB_P_OUTPUT_EMPTY:
 566                 /* the adapter got it */
 567                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,
 568                         "out empty:%1d %02x", q->nr, count);
 569
 570                 atomic_sub(count, &q->nr_buf_used);
 571                 if (q->irq_ptr->perf_stat_enabled)
 572                         account_sbals(q, count);
 573                 return count;
 574         case SLSB_P_OUTPUT_ERROR:
 575                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x",
 576                               q->nr, count);
 577
 578                 *error = QDIO_ERROR_SLSB_STATE;
 579                 process_buffer_error(q, start, count);
 580                 atomic_sub(count, &q->nr_buf_used);
 581                 if (q->irq_ptr->perf_stat_enabled)
 582                         account_sbals_error(q, count);
 583                 return count;
 584         case SLSB_CU_OUTPUT_PRIMED:
 585                 /* the adapter has not fetched the output yet */
 586                 if (q->irq_ptr->perf_stat_enabled)
 587                         q->q_stats.nr_sbal_nop++;
 588                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
 589                               q->nr);
 590                 return 0;
 591         case SLSB_P_OUTPUT_HALTED:
 592                 return 0;
 593         case SLSB_P_OUTPUT_NOT_INIT:
 594                 /* We should never see this state, throw a WARN: */
 595         default:
 596                 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
 597                               "found state %#x at index %u on queue %u\n",
 598                               state, start, q->nr);
 599                 return 0;
 600         }
 601 }
 602
 603 int qdio_inspect_output_queue(struct ccw_device *cdev, unsigned int nr,
 604                               unsigned int *bufnr, unsigned int *error)
 605 {
 606         struct qdio_irq *irq = cdev->private->qdio_data;
 607         unsigned int start;
 608         struct qdio_q *q;
 609         int count;
 610
 611         if (!irq)
 612                 return -ENODEV;
 613
 614         q = irq->output_qs[nr];
 615         start = q->first_to_check;
 616         *error = 0;
 617
 618         count = get_outbound_buffer_frontier(q, start, error);
 619         if (count == 0)
 620                 return 0;
 621
 622         *bufnr = start;
 623         q->first_to_check = add_buf(start, count);
 624         return count;
 625 }
 626 EXPORT_SYMBOL_GPL(qdio_inspect_output_queue);
 627
 628 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,
 629                                 dma64_t aob)
 630 {
 631         int retries = 0, cc;
 632         unsigned int busy_bit;
 633
 634         if (!qdio_need_siga_out(q->irq_ptr))
 635                 return 0;
 636
 637         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
 638 retry:
 639         qperf_inc(q, siga_write);
 640
 641         cc = qdio_siga_output(q, count, &busy_bit, aob);
 642         switch (cc) {
 643         case 0:
 644                 break;
 645         case 2:
 646                 if (busy_bit) {
 647                         while (++retries < QDIO_BUSY_BIT_RETRIES) {
 648                                 mdelay(QDIO_BUSY_BIT_RETRY_DELAY);
 649                                 goto retry;
 650                         }
 651                         DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);
 652                         cc = -EBUSY;
 653                 } else {
 654                         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
 655                         cc = -ENOBUFS;
 656                 }
 657                 break;
 658         case 1:
 659         case 3:
 660                 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
 661                 cc = -EIO;
 662                 break;
 663         }
 664         if (retries) {
 665                 DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);
 666                 DBF_ERROR("count:%u", retries);
 667         }
 668         return cc;
 669 }
 670
 671 static inline void qdio_set_state(struct qdio_irq *irq_ptr,
 672                                   enum qdio_irq_states state)
 673 {
 674         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
 675
 676         irq_ptr->state = state;
 677         mb();
 678 }
 679
 680 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
 681 {
 682         if (irb->esw.esw0.erw.cons) {
 683                 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
 684                 DBF_ERROR_HEX(irb, 64);
 685                 DBF_ERROR_HEX(irb->ecw, 64);
 686         }
 687 }
 688
 689 /* PCI interrupt handler */
 690 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
 691 {
 692         if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
 693                 return;
 694
 695         qdio_deliver_irq(irq_ptr);
 696         irq_ptr->last_data_irq_time = get_lowcore()->int_clock;
 697 }
 698
 699 static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,
 700                                        unsigned long intparm, int cstat,
 701                                        int dstat)
 702 {
 703         unsigned int first_to_check = 0;
 704
 705         DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
 706         DBF_ERROR("intp :%lx", intparm);
 707         DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
 708
 709         /* zfcp wants this: */
 710         if (irq_ptr->nr_input_qs)
 711                 first_to_check = irq_ptr->input_qs[0]->first_to_check;
 712
 713         irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0,
 714                                first_to_check, 0, irq_ptr->int_parm);
 715         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
 716         /*
 717          * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen.
 718          * Therefore we call the LGR detection function here.
 719          */
 720         lgr_info_log();
 721 }
 722
 723 static int qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat,
 724                                      int dstat, int dcc)
 725 {
 726         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
 727
 728         if (cstat)
 729                 goto error;
 730         if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
 731                 goto error;
 732         if (dcc == 1)
 733                 return -EAGAIN;
 734         if (!(dstat & DEV_STAT_DEV_END))
 735                 goto error;
 736         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
 737         return 0;
 738
 739 error:
 740         DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
 741         DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
 742         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 743         return -EIO;
 744 }
 745
 746 /* qdio interrupt handler */
 747 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
 748                       struct irb *irb)
 749 {
 750         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 751         struct subchannel_id schid;
 752         int cstat, dstat, rc, dcc;
 753
 754         if (!intparm || !irq_ptr) {
 755                 ccw_device_get_schid(cdev, &schid);
 756                 DBF_ERROR("qint:%4x", schid.sch_no);
 757                 return;
 758         }
 759
 760         if (irq_ptr->perf_stat_enabled)
 761                 irq_ptr->perf_stat.qdio_int++;
 762
 763         if (IS_ERR(irb)) {
 764                 DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
 765                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 766                 wake_up(&cdev->private->wait_q);
 767                 return;
 768         }
 769         qdio_irq_check_sense(irq_ptr, irb);
 770         cstat = irb->scsw.cmd.cstat;
 771         dstat = irb->scsw.cmd.dstat;
 772         dcc   = scsw_cmd_is_valid_cc(&irb->scsw) ? irb->scsw.cmd.cc : 0;
 773         rc    = 0;
 774
 775         switch (irq_ptr->state) {
 776         case QDIO_IRQ_STATE_INACTIVE:
 777                 rc = qdio_establish_handle_irq(irq_ptr, cstat, dstat, dcc);
 778                 break;
 779         case QDIO_IRQ_STATE_CLEANUP:
 780                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 781                 break;
 782         case QDIO_IRQ_STATE_ESTABLISHED:
 783         case QDIO_IRQ_STATE_ACTIVE:
 784                 if (cstat & SCHN_STAT_PCI) {
 785                         qdio_int_handler_pci(irq_ptr);
 786                         return;
 787                 }
 788                 if (cstat || dstat)
 789                         qdio_handle_activate_check(irq_ptr, intparm, cstat,
 790                                                    dstat);
 791                 else if (dcc == 1)
 792                         rc = -EAGAIN;
 793                 break;
 794         case QDIO_IRQ_STATE_STOPPED:
 795                 break;
 796         default:
 797                 WARN_ON_ONCE(1);
 798         }
 799
 800         if (rc == -EAGAIN) {
 801                 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qint retry");
 802                 rc = ccw_device_start(cdev, irq_ptr->ccw, intparm, 0, 0);
 803                 if (!rc)
 804                         return;
 805                 DBF_ERROR("%4x RETRY ERR", irq_ptr->schid.sch_no);
 806                 DBF_ERROR("rc:%4x", rc);
 807                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 808         }
 809
 810         wake_up(&cdev->private->wait_q);
 811 }
 812
 813 /**
 814  * qdio_get_ssqd_desc - get qdio subchannel description
 815  * @cdev: ccw device to get description for
 816  * @data: where to store the ssqd
 817  *
 818  * Returns 0 or an error code. The results of the chsc are stored in the
 819  * specified structure.
 820  */
 821 int qdio_get_ssqd_desc(struct ccw_device *cdev,
 822                        struct qdio_ssqd_desc *data)
 823 {
 824         struct subchannel_id schid;
 825
 826         if (!cdev || !cdev->private)
 827                 return -EINVAL;
 828
 829         ccw_device_get_schid(cdev, &schid);
 830         DBF_EVENT("get ssqd:%4x", schid.sch_no);
 831         return qdio_setup_get_ssqd(NULL, &schid, data);
 832 }
 833 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
 834
 835 static int qdio_cancel_ccw(struct qdio_irq *irq, int how)
 836 {
 837         struct ccw_device *cdev = irq->cdev;
 838         long timeout;
 839         int rc;
 840
 841         spin_lock_irq(get_ccwdev_lock(cdev));
 842         qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP);
 843         if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
 844                 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
 845         else
 846                 /* default behaviour is halt */
 847                 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
 848         spin_unlock_irq(get_ccwdev_lock(cdev));
 849         if (rc) {
 850                 DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no);
 851                 DBF_ERROR("rc:%4d", rc);
 852                 return rc;
 853         }
 854
 855         timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
 856                                                    irq->state == QDIO_IRQ_STATE_INACTIVE ||
 857                                                    irq->state == QDIO_IRQ_STATE_ERR,
 858                                                    10 * HZ);
 859         if (timeout <= 0)
 860                 rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
 861
 862         return rc;
 863 }
 864
 865 /**
 866  * qdio_shutdown - shut down a qdio subchannel
 867  * @cdev: associated ccw device
 868  * @how: use halt or clear to shutdown
 869  */
 870 int qdio_shutdown(struct ccw_device *cdev, int how)
 871 {
 872         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 873         struct subchannel_id schid;
 874         int rc;
 875
 876         if (!irq_ptr)
 877                 return -ENODEV;
 878
 879         WARN_ON_ONCE(irqs_disabled());
 880         ccw_device_get_schid(cdev, &schid);
 881         DBF_EVENT("qshutdown:%4x", schid.sch_no);
 882
 883         mutex_lock(&irq_ptr->setup_mutex);
 884         /*
 885          * Subchannel was already shot down. We cannot prevent being called
 886          * twice since cio may trigger a shutdown asynchronously.
 887          */
 888         if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
 889                 mutex_unlock(&irq_ptr->setup_mutex);
 890                 return 0;
 891         }
 892
 893         /*
 894          * Indicate that the device is going down.
 895          */
 896         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
 897
 898         qdio_shutdown_debug_entries(irq_ptr);
 899
 900         rc = qdio_cancel_ccw(irq_ptr, how);
 901         qdio_shutdown_thinint(irq_ptr);
 902         qdio_shutdown_irq(irq_ptr);
 903
 904         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 905         mutex_unlock(&irq_ptr->setup_mutex);
 906         if (rc)
 907                 return rc;
 908         return 0;
 909 }
 910 EXPORT_SYMBOL_GPL(qdio_shutdown);
 911
 912 /**
 913  * qdio_free - free data structures for a qdio subchannel
 914  * @cdev: associated ccw device
 915  */
 916 int qdio_free(struct ccw_device *cdev)
 917 {
 918         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 919         struct subchannel_id schid;
 920
 921         if (!irq_ptr)
 922                 return -ENODEV;
 923
 924         ccw_device_get_schid(cdev, &schid);
 925         DBF_EVENT("qfree:%4x", schid.sch_no);
 926         DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned");
 927         mutex_lock(&irq_ptr->setup_mutex);
 928
 929         irq_ptr->debug_area = NULL;
 930         cdev->private->qdio_data = NULL;
 931         mutex_unlock(&irq_ptr->setup_mutex);
 932
 933         qdio_free_queues(irq_ptr);
 934         free_page((unsigned long) irq_ptr->qdr);
 935         free_page(irq_ptr->chsc_page);
 936         kfree(irq_ptr->ccw);
 937         free_page((unsigned long) irq_ptr);
 938         return 0;
 939 }
 940 EXPORT_SYMBOL_GPL(qdio_free);
 941
 942 /**
 943  * qdio_allocate - allocate qdio queues and associated data
 944  * @cdev: associated ccw device
 945  * @no_input_qs: allocate this number of Input Queues
 946  * @no_output_qs: allocate this number of Output Queues
 947  */
 948 int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 949                   unsigned int no_output_qs)
 950 {
 951         struct subchannel_id schid;
 952         struct qdio_irq *irq_ptr;
 953         int rc = -ENOMEM;
 954
 955         ccw_device_get_schid(cdev, &schid);
 956         DBF_EVENT("qallocate:%4x", schid.sch_no);
 957
 958         if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ ||
 959             no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)
 960                 return -EINVAL;
 961
 962         irq_ptr = (void *) get_zeroed_page(GFP_KERNEL);
 963         if (!irq_ptr)
 964                 return -ENOMEM;
 965
 966         irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA);
 967         if (!irq_ptr->ccw)
 968                 goto err_ccw;
 969
 970         /* kmemleak doesn't scan the page-allocated irq_ptr: */
 971         kmemleak_not_leak(irq_ptr->ccw);
 972
 973         irq_ptr->cdev = cdev;
 974         mutex_init(&irq_ptr->setup_mutex);
 975         if (qdio_allocate_dbf(irq_ptr))
 976                 goto err_dbf;
 977
 978         DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,
 979                       no_output_qs);
 980
 981         /*
 982          * Allocate a page for the chsc calls in qdio_establish.
 983          * Must be pre-allocated since a zfcp recovery will call
 984          * qdio_establish. In case of low memory and swap on a zfcp disk
 985          * we may not be able to allocate memory otherwise.
 986          */
 987         irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
 988         if (!irq_ptr->chsc_page)
 989                 goto err_chsc;
 990
 991         /* qdr is used in ccw1.cda which is u32 */
 992         irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 993         if (!irq_ptr->qdr)
 994                 goto err_qdr;
 995
 996         rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);
 997         if (rc)
 998                 goto err_queues;
 999
1000         cdev->private->qdio_data = irq_ptr;
1001         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1002         return 0;
1003
1004 err_queues:
1005         free_page((unsigned long) irq_ptr->qdr);
1006 err_qdr:
1007         free_page(irq_ptr->chsc_page);
1008 err_chsc:
1009 err_dbf:
1010         kfree(irq_ptr->ccw);
1011 err_ccw:
1012         free_page((unsigned long) irq_ptr);
1013         return rc;
1014 }
1015 EXPORT_SYMBOL_GPL(qdio_allocate);
1016
1017 static void qdio_trace_init_data(struct qdio_irq *irq,
1018                                  struct qdio_initialize *data)
1019 {
1020         DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format);
1021         DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format);
1022         DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR);
1023         DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs,
1024                       data->no_output_qs);
1025         DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR);
1026         DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR);
1027         DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR);
1028         DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR);
1029         DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *),
1030                     DBF_ERR);
1031 }
1032
1033 /**
1034  * qdio_establish - establish queues on a qdio subchannel
1035  * @cdev: associated ccw device
1036  * @init_data: initialization data
1037  */
1038 int qdio_establish(struct ccw_device *cdev,
1039                    struct qdio_initialize *init_data)
1040 {
1041         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1042         struct subchannel_id schid;
1043         struct ciw *ciw;
1044         long timeout;
1045         int rc;
1046
1047         ccw_device_get_schid(cdev, &schid);
1048         DBF_EVENT("qestablish:%4x", schid.sch_no);
1049
1050         if (!irq_ptr)
1051                 return -ENODEV;
1052
1053         if (init_data->no_input_qs > irq_ptr->max_input_qs ||
1054             init_data->no_output_qs > irq_ptr->max_output_qs)
1055                 return -EINVAL;
1056
1057         /* Needed as error_handler: */
1058         if (!init_data->input_handler)
1059                 return -EINVAL;
1060
1061         if (init_data->no_output_qs && !init_data->output_handler)
1062                 return -EINVAL;
1063
1064         if (!init_data->input_sbal_addr_array ||
1065             !init_data->output_sbal_addr_array)
1066                 return -EINVAL;
1067
1068         if (!init_data->irq_poll)
1069                 return -EINVAL;
1070
1071         ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE);
1072         if (!ciw) {
1073                 DBF_ERROR("%4x NO EQ", schid.sch_no);
1074                 return -EIO;
1075         }
1076
1077         mutex_lock(&irq_ptr->setup_mutex);
1078         qdio_trace_init_data(irq_ptr, init_data);
1079         qdio_setup_irq(irq_ptr, init_data);
1080
1081         rc = qdio_establish_thinint(irq_ptr);
1082         if (rc)
1083                 goto err_thinint;
1084
1085         /* establish q */
1086         irq_ptr->ccw->cmd_code = ciw->cmd;
1087         irq_ptr->ccw->flags = CCW_FLAG_SLI;
1088         irq_ptr->ccw->count = ciw->count;
1089         irq_ptr->ccw->cda = virt_to_dma32(irq_ptr->qdr);
1090
1091         spin_lock_irq(get_ccwdev_lock(cdev));
1092         ccw_device_set_options_mask(cdev, 0);
1093
1094         rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
1095         spin_unlock_irq(get_ccwdev_lock(cdev));
1096         if (rc) {
1097                 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1098                 DBF_ERROR("rc:%4x", rc);
1099                 goto err_ccw_start;
1100         }
1101
1102         timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
1103                                                    irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
1104                                                    irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
1105         if (timeout <= 0) {
1106                 rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
1107                 goto err_ccw_timeout;
1108         }
1109
1110         if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1111                 rc = -EIO;
1112                 goto err_ccw_error;
1113         }
1114
1115         qdio_setup_ssqd_info(irq_ptr);
1116
1117         /* qebsm is now setup if available, initialize buffer states */
1118         qdio_init_buf_states(irq_ptr);
1119
1120         mutex_unlock(&irq_ptr->setup_mutex);
1121         qdio_print_subchannel_info(irq_ptr);
1122         qdio_setup_debug_entries(irq_ptr);
1123         return 0;
1124
1125 err_ccw_timeout:
1126         qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR);
1127 err_ccw_error:
1128 err_ccw_start:
1129         qdio_shutdown_thinint(irq_ptr);
1130 err_thinint:
1131         qdio_shutdown_irq(irq_ptr);
1132         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1133         mutex_unlock(&irq_ptr->setup_mutex);
1134         return rc;
1135 }
1136 EXPORT_SYMBOL_GPL(qdio_establish);
1137
1138 /**
1139  * qdio_activate - activate queues on a qdio subchannel
1140  * @cdev: associated cdev
1141  */
1142 int qdio_activate(struct ccw_device *cdev)
1143 {
1144         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1145         struct subchannel_id schid;
1146         struct ciw *ciw;
1147         int rc;
1148
1149         ccw_device_get_schid(cdev, &schid);
1150         DBF_EVENT("qactivate:%4x", schid.sch_no);
1151
1152         if (!irq_ptr)
1153                 return -ENODEV;
1154
1155         ciw = ccw_device_get_ciw(cdev, CIW_TYPE_AQUEUE);
1156         if (!ciw) {
1157                 DBF_ERROR("%4x NO AQ", schid.sch_no);
1158                 return -EIO;
1159         }
1160
1161         mutex_lock(&irq_ptr->setup_mutex);
1162         if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1163                 rc = -EBUSY;
1164                 goto out;
1165         }
1166
1167         irq_ptr->ccw->cmd_code = ciw->cmd;
1168         irq_ptr->ccw->flags = CCW_FLAG_SLI;
1169         irq_ptr->ccw->count = ciw->count;
1170         irq_ptr->ccw->cda = 0;
1171
1172         spin_lock_irq(get_ccwdev_lock(cdev));
1173         ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
1174
1175         rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE,
1176                               0, DOIO_DENY_PREFETCH);
1177         spin_unlock_irq(get_ccwdev_lock(cdev));
1178         if (rc) {
1179                 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
1180                 DBF_ERROR("rc:%4x", rc);
1181                 goto out;
1182         }
1183
1184         /* wait for subchannel to become active */
1185         msleep(5);
1186
1187         switch (irq_ptr->state) {
1188         case QDIO_IRQ_STATE_STOPPED:
1189         case QDIO_IRQ_STATE_ERR:
1190                 rc = -EIO;
1191                 break;
1192         default:
1193                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
1194                 rc = 0;
1195         }
1196 out:
1197         mutex_unlock(&irq_ptr->setup_mutex);
1198         return rc;
1199 }
1200 EXPORT_SYMBOL_GPL(qdio_activate);
1201
1202 /**
1203  * handle_inbound - reset processed input buffers
1204  * @q: queue containing the buffers
1205  * @bufnr: first buffer to process
1206  * @count: how many buffers are emptied
1207  */
1208 static int handle_inbound(struct qdio_q *q, int bufnr, int count)
1209 {
1210         int overlap;
1211
1212         qperf_inc(q, inbound_call);
1213
1214         /* If any processed SBALs are returned to HW, adjust our tracking: */
1215         overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),
1216                              q->u.in.batch_count);
1217         if (overlap > 0) {
1218                 q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);
1219                 q->u.in.batch_count -= overlap;
1220         }
1221
1222         count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1223         atomic_add(count, &q->nr_buf_used);
1224
1225         if (qdio_need_siga_in(q->irq_ptr))
1226                 return qdio_siga_input(q);
1227
1228         return 0;
1229 }
1230
1231 /**
1232  * qdio_add_bufs_to_input_queue - process buffers on an Input Queue
1233  * @cdev: associated ccw_device for the qdio subchannel
1234  * @q_nr: queue number
1235  * @bufnr: buffer number
1236  * @count: how many buffers to process
1237  */
1238 int qdio_add_bufs_to_input_queue(struct ccw_device *cdev, unsigned int q_nr,
1239                                  unsigned int bufnr, unsigned int count)
1240 {
1241         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1242
1243         if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1244                 return -EINVAL;
1245
1246         if (!irq_ptr)
1247                 return -ENODEV;
1248
1249         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count);
1250
1251         if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1252                 return -EIO;
1253         if (!count)
1254                 return 0;
1255
1256         return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count);
1257 }
1258 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_input_queue);
1259
1260 /**
1261  * handle_outbound - process filled outbound buffers
1262  * @q: queue containing the buffers
1263  * @bufnr: first buffer to process
1264  * @count: how many buffers are filled
1265  * @aob: asynchronous operation block
1266  */
1267 static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count,
1268                            struct qaob *aob)
1269 {
1270         unsigned char state = 0;
1271         int used, rc = 0;
1272
1273         qperf_inc(q, outbound_call);
1274
1275         count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1276         used = atomic_add_return(count, &q->nr_buf_used);
1277
1278         if (used == QDIO_MAX_BUFFERS_PER_Q)
1279                 qperf_inc(q, outbound_queue_full);
1280
1281         if (queue_type(q) == QDIO_IQDIO_QFMT) {
1282                 dma64_t phys_aob = aob ? virt_to_dma64(aob) : 0;
1283
1284                 WARN_ON_ONCE(!IS_ALIGNED(dma64_to_u64(phys_aob), 256));
1285                 rc = qdio_kick_outbound_q(q, count, phys_aob);
1286         } else if (qdio_need_siga_sync(q->irq_ptr)) {
1287                 rc = qdio_sync_output_queue(q);
1288         } else if (count < QDIO_MAX_BUFFERS_PER_Q &&
1289                    get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 &&
1290                    state == SLSB_CU_OUTPUT_PRIMED) {
1291                 /* The previous buffer is not processed yet, tack on. */
1292                 qperf_inc(q, fast_requeue);
1293         } else {
1294                 rc = qdio_kick_outbound_q(q, count, 0);
1295         }
1296
1297         return rc;
1298 }
1299
1300 /**
1301  * qdio_add_bufs_to_output_queue - process buffers on an Output Queue
1302  * @cdev: associated ccw_device for the qdio subchannel
1303  * @q_nr: queue number
1304  * @bufnr: buffer number
1305  * @count: how many buffers to process
1306  * @aob: asynchronous operation block
1307  */
1308 int qdio_add_bufs_to_output_queue(struct ccw_device *cdev, unsigned int q_nr,
1309                                   unsigned int bufnr, unsigned int count,
1310                                   struct qaob *aob)
1311 {
1312         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1313
1314         if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1315                 return -EINVAL;
1316
1317         if (!irq_ptr)
1318                 return -ENODEV;
1319
1320         DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count);
1321
1322         if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1323                 return -EIO;
1324         if (!count)
1325                 return 0;
1326
1327         return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob);
1328 }
1329 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_output_queue);
1330
1331 /**
1332  * qdio_start_irq - enable interrupt processing for the device
1333  * @cdev: associated ccw_device for the qdio subchannel
1334  *
1335  * Return codes
1336  *   0 - success
1337  *   1 - irqs not started since new data is available
1338  */
1339 int qdio_start_irq(struct ccw_device *cdev)
1340 {
1341         struct qdio_q *q;
1342         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1343         unsigned int i;
1344
1345         if (!irq_ptr)
1346                 return -ENODEV;
1347
1348         for_each_input_queue(irq_ptr, q, i)
1349                 qdio_stop_polling(q);
1350
1351         clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state);
1352
1353         /*
1354          * We need to check again to not lose initiative after
1355          * resetting the ACK state.
1356          */
1357         if (test_nonshared_ind(irq_ptr))
1358                 goto rescan;
1359
1360         for_each_input_queue(irq_ptr, q, i) {
1361                 if (!qdio_inbound_q_done(q, q->first_to_check))
1362                         goto rescan;
1363         }
1364
1365         return 0;
1366
1367 rescan:
1368         if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1369                 return 0;
1370         else
1371                 return 1;
1372
1373 }
1374 EXPORT_SYMBOL(qdio_start_irq);
1375
1376 /**
1377  * qdio_stop_irq - disable interrupt processing for the device
1378  * @cdev: associated ccw_device for the qdio subchannel
1379  *
1380  * Return codes
1381  *   0 - interrupts were already disabled
1382  *   1 - interrupts successfully disabled
1383  */
1384 int qdio_stop_irq(struct ccw_device *cdev)
1385 {
1386         struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1387
1388         if (!irq_ptr)
1389                 return -ENODEV;
1390
1391         if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1392                 return 0;
1393         else
1394                 return 1;
1395 }
1396 EXPORT_SYMBOL(qdio_stop_irq);
1397
1398 static int __init init_QDIO(void)
1399 {
1400         int rc;
1401
1402         rc = qdio_debug_init();
1403         if (rc)
1404                 return rc;
1405         rc = qdio_setup_init();
1406         if (rc)
1407                 goto out_debug;
1408         rc = qdio_thinint_init();
1409         if (rc)
1410                 goto out_cache;
1411         return 0;
1412
1413 out_cache:
1414         qdio_setup_exit();
1415 out_debug:
1416         qdio_debug_exit();
1417         return rc;
1418 }
1419
1420 static void __exit exit_QDIO(void)
1421 {
1422         qdio_thinint_exit();
1423         qdio_setup_exit();
1424         qdio_debug_exit();
1425 }
1426
1427 module_init(init_QDIO);
1428 module_exit(exit_QDIO);