hw/block/dataplane/xen-block.c

   1 /*
   2  * Copyright (c) 2018  Citrix Systems Inc.
   3  * (c) Gerd Hoffmann <[email protected]>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; under version 2 of the License.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with this program; if not, see <http://www.gnu.org/licenses/>.
  16  *
  17  * Contributions after 2012-01-13 are licensed under the terms of the
  18  * GNU GPL, version 2 or (at your option) any later version.
  19  */
  20
  21 #include "qemu/osdep.h"
  22 #include "qemu/error-report.h"
  23 #include "qapi/error.h"
  24 #include "hw/hw.h"
  25 #include "hw/xen/xen_common.h"
  26 #include "hw/block/xen_blkif.h"
  27 #include "sysemu/block-backend.h"
  28 #include "sysemu/iothread.h"
  29 #include "xen-block.h"
  30
  31 typedef struct XenBlockRequest {
  32     blkif_request_t req;
  33     int16_t status;
  34     off_t start;
  35     QEMUIOVector v;
  36     void *buf;
  37     size_t size;
  38     int presync;
  39     int aio_inflight;
  40     int aio_errors;
  41     XenBlockDataPlane *dataplane;
  42     QLIST_ENTRY(XenBlockRequest) list;
  43     BlockAcctCookie acct;
  44 } XenBlockRequest;
  45
  46 struct XenBlockDataPlane {
  47     XenDevice *xendev;
  48     XenEventChannel *event_channel;
  49     unsigned int *ring_ref;
  50     unsigned int nr_ring_ref;
  51     void *sring;
  52     int protocol;
  53     blkif_back_rings_t rings;
  54     int more_work;
  55     QLIST_HEAD(inflight_head, XenBlockRequest) inflight;
  56     QLIST_HEAD(freelist_head, XenBlockRequest) freelist;
  57     int requests_total;
  58     int requests_inflight;
  59     unsigned int max_requests;
  60     BlockBackend *blk;
  61     QEMUBH *bh;
  62     IOThread *iothread;
  63     AioContext *ctx;
  64 };
  65
  66 static void reset_request(XenBlockRequest *request)
  67 {
  68     memset(&request->req, 0, sizeof(request->req));
  69     request->status = 0;
  70     request->start = 0;
  71     request->size = 0;
  72     request->presync = 0;
  73
  74     request->aio_inflight = 0;
  75     request->aio_errors = 0;
  76
  77     request->dataplane = NULL;
  78     memset(&request->list, 0, sizeof(request->list));
  79     memset(&request->acct, 0, sizeof(request->acct));
  80
  81     qemu_iovec_reset(&request->v);
  82 }
  83
  84 static XenBlockRequest *xen_block_start_request(XenBlockDataPlane *dataplane)
  85 {
  86     XenBlockRequest *request = NULL;
  87
  88     if (QLIST_EMPTY(&dataplane->freelist)) {
  89         if (dataplane->requests_total >= dataplane->max_requests) {
  90             goto out;
  91         }
  92         /* allocate new struct */
  93         request = g_malloc0(sizeof(*request));
  94         request->dataplane = dataplane;
  95         /*
  96          * We cannot need more pages per requests than this, and since we
  97          * re-use requests, allocate the memory once here. It will be freed
  98          * xen_block_dataplane_destroy() when the request list is freed.
  99          */
 100         request->buf = qemu_memalign(XC_PAGE_SIZE,
 101                                      BLKIF_MAX_SEGMENTS_PER_REQUEST *
 102                                      XC_PAGE_SIZE);
 103         dataplane->requests_total++;
 104         qemu_iovec_init(&request->v, 1);
 105     } else {
 106         /* get one from freelist */
 107         request = QLIST_FIRST(&dataplane->freelist);
 108         QLIST_REMOVE(request, list);
 109     }
 110     QLIST_INSERT_HEAD(&dataplane->inflight, request, list);
 111     dataplane->requests_inflight++;
 112
 113 out:
 114     return request;
 115 }
 116
 117 static void xen_block_finish_request(XenBlockRequest *request)
 118 {
 119     XenBlockDataPlane *dataplane = request->dataplane;
 120
 121     QLIST_REMOVE(request, list);
 122     dataplane->requests_inflight--;
 123 }
 124
 125 static void xen_block_release_request(XenBlockRequest *request)
 126 {
 127     XenBlockDataPlane *dataplane = request->dataplane;
 128
 129     QLIST_REMOVE(request, list);
 130     reset_request(request);
 131     request->dataplane = dataplane;
 132     QLIST_INSERT_HEAD(&dataplane->freelist, request, list);
 133     dataplane->requests_inflight--;
 134 }
 135
 136 /*
 137  * translate request into iovec + start offset
 138  * do sanity checks along the way
 139  */
 140 static int xen_block_parse_request(XenBlockRequest *request)
 141 {
 142     XenBlockDataPlane *dataplane = request->dataplane;
 143     size_t len;
 144     int i;
 145
 146     switch (request->req.operation) {
 147     case BLKIF_OP_READ:
 148         break;
 149     case BLKIF_OP_FLUSH_DISKCACHE:
 150         request->presync = 1;
 151         if (!request->req.nr_segments) {
 152             return 0;
 153         }
 154         /* fall through */
 155     case BLKIF_OP_WRITE:
 156         break;
 157     case BLKIF_OP_DISCARD:
 158         return 0;
 159     default:
 160         error_report("error: unknown operation (%d)", request->req.operation);
 161         goto err;
 162     };
 163
 164     if (request->req.operation != BLKIF_OP_READ &&
 165         blk_is_read_only(dataplane->blk)) {
 166         error_report("error: write req for ro device");
 167         goto err;
 168     }
 169
 170     request->start = request->req.sector_number * XEN_BLKIF_SECTOR_SIZE;
 171     for (i = 0; i < request->req.nr_segments; i++) {
 172         if (i == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
 173             error_report("error: nr_segments too big");
 174             goto err;
 175         }
 176         if (request->req.seg[i].first_sect > request->req.seg[i].last_sect) {
 177             error_report("error: first > last sector");
 178             goto err;
 179         }
 180         if (request->req.seg[i].last_sect * XEN_BLKIF_SECTOR_SIZE >=
 181             XC_PAGE_SIZE) {
 182             error_report("error: page crossing");
 183             goto err;
 184         }
 185
 186         len = (request->req.seg[i].last_sect -
 187                request->req.seg[i].first_sect + 1) * XEN_BLKIF_SECTOR_SIZE;
 188         request->size += len;
 189     }
 190     if (request->start + request->size > blk_getlength(dataplane->blk)) {
 191         error_report("error: access beyond end of file");
 192         goto err;
 193     }
 194     return 0;
 195
 196 err:
 197     request->status = BLKIF_RSP_ERROR;
 198     return -1;
 199 }
 200
 201 static int xen_block_copy_request(XenBlockRequest *request)
 202 {
 203     XenBlockDataPlane *dataplane = request->dataplane;
 204     XenDevice *xendev = dataplane->xendev;
 205     XenDeviceGrantCopySegment segs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 206     int i, count;
 207     bool to_domain = (request->req.operation == BLKIF_OP_READ);
 208     void *virt = request->buf;
 209     Error *local_err = NULL;
 210
 211     if (request->req.nr_segments == 0) {
 212         return 0;
 213     }
 214
 215     count = request->req.nr_segments;
 216
 217     for (i = 0; i < count; i++) {
 218         if (to_domain) {
 219             segs[i].dest.foreign.ref = request->req.seg[i].gref;
 220             segs[i].dest.foreign.offset = request->req.seg[i].first_sect *
 221                 XEN_BLKIF_SECTOR_SIZE;
 222             segs[i].source.virt = virt;
 223         } else {
 224             segs[i].source.foreign.ref = request->req.seg[i].gref;
 225             segs[i].source.foreign.offset = request->req.seg[i].first_sect *
 226                 XEN_BLKIF_SECTOR_SIZE;
 227             segs[i].dest.virt = virt;
 228         }
 229         segs[i].len = (request->req.seg[i].last_sect -
 230                        request->req.seg[i].first_sect + 1) *
 231                       XEN_BLKIF_SECTOR_SIZE;
 232         virt += segs[i].len;
 233     }
 234
 235     xen_device_copy_grant_refs(xendev, to_domain, segs, count, &local_err);
 236
 237     if (local_err) {
 238         error_reportf_err(local_err, "failed to copy data: ");
 239
 240         request->aio_errors++;
 241         return -1;
 242     }
 243
 244     return 0;
 245 }
 246
 247 static int xen_block_do_aio(XenBlockRequest *request);
 248 static int xen_block_send_response(XenBlockRequest *request);
 249
 250 static void xen_block_complete_aio(void *opaque, int ret)
 251 {
 252     XenBlockRequest *request = opaque;
 253     XenBlockDataPlane *dataplane = request->dataplane;
 254
 255     aio_context_acquire(dataplane->ctx);
 256
 257     if (ret != 0) {
 258         error_report("%s I/O error",
 259                      request->req.operation == BLKIF_OP_READ ?
 260                      "read" : "write");
 261         request->aio_errors++;
 262     }
 263
 264     request->aio_inflight--;
 265     if (request->presync) {
 266         request->presync = 0;
 267         xen_block_do_aio(request);
 268         goto done;
 269     }
 270     if (request->aio_inflight > 0) {
 271         goto done;
 272     }
 273
 274     switch (request->req.operation) {
 275     case BLKIF_OP_READ:
 276         /* in case of failure request->aio_errors is increased */
 277         if (ret == 0) {
 278             xen_block_copy_request(request);
 279         }
 280         break;
 281     case BLKIF_OP_WRITE:
 282     case BLKIF_OP_FLUSH_DISKCACHE:
 283     default:
 284         break;
 285     }
 286
 287     request->status = request->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
 288     xen_block_finish_request(request);
 289
 290     switch (request->req.operation) {
 291     case BLKIF_OP_WRITE:
 292     case BLKIF_OP_FLUSH_DISKCACHE:
 293         if (!request->req.nr_segments) {
 294             break;
 295         }
 296         /* fall through */
 297     case BLKIF_OP_READ:
 298         if (request->status == BLKIF_RSP_OKAY) {
 299             block_acct_done(blk_get_stats(dataplane->blk), &request->acct);
 300         } else {
 301             block_acct_failed(blk_get_stats(dataplane->blk), &request->acct);
 302         }
 303         break;
 304     case BLKIF_OP_DISCARD:
 305     default:
 306         break;
 307     }
 308     if (xen_block_send_response(request)) {
 309         Error *local_err = NULL;
 310
 311         xen_device_notify_event_channel(dataplane->xendev,
 312                                         dataplane->event_channel,
 313                                         &local_err);
 314         if (local_err) {
 315             error_report_err(local_err);
 316         }
 317     }
 318     xen_block_release_request(request);
 319
 320     qemu_bh_schedule(dataplane->bh);
 321
 322 done:
 323     aio_context_release(dataplane->ctx);
 324 }
 325
 326 static bool xen_block_split_discard(XenBlockRequest *request,
 327                                     blkif_sector_t sector_number,
 328                                     uint64_t nr_sectors)
 329 {
 330     XenBlockDataPlane *dataplane = request->dataplane;
 331     int64_t byte_offset;
 332     int byte_chunk;
 333     uint64_t byte_remaining;
 334     uint64_t sec_start = sector_number;
 335     uint64_t sec_count = nr_sectors;
 336
 337     /* Wrap around, or overflowing byte limit? */
 338     if (sec_start + sec_count < sec_count ||
 339         sec_start + sec_count > INT64_MAX / XEN_BLKIF_SECTOR_SIZE) {
 340         return false;
 341     }
 342
 343     byte_offset = sec_start * XEN_BLKIF_SECTOR_SIZE;
 344     byte_remaining = sec_count * XEN_BLKIF_SECTOR_SIZE;
 345
 346     do {
 347         byte_chunk = byte_remaining > BDRV_REQUEST_MAX_BYTES ?
 348             BDRV_REQUEST_MAX_BYTES : byte_remaining;
 349         request->aio_inflight++;
 350         blk_aio_pdiscard(dataplane->blk, byte_offset, byte_chunk,
 351                          xen_block_complete_aio, request);
 352         byte_remaining -= byte_chunk;
 353         byte_offset += byte_chunk;
 354     } while (byte_remaining > 0);
 355
 356     return true;
 357 }
 358
 359 static int xen_block_do_aio(XenBlockRequest *request)
 360 {
 361     XenBlockDataPlane *dataplane = request->dataplane;
 362
 363     if (request->req.nr_segments &&
 364         (request->req.operation == BLKIF_OP_WRITE ||
 365          request->req.operation == BLKIF_OP_FLUSH_DISKCACHE) &&
 366         xen_block_copy_request(request)) {
 367         goto err;
 368     }
 369
 370     request->aio_inflight++;
 371     if (request->presync) {
 372         blk_aio_flush(request->dataplane->blk, xen_block_complete_aio,
 373                       request);
 374         return 0;
 375     }
 376
 377     switch (request->req.operation) {
 378     case BLKIF_OP_READ:
 379         qemu_iovec_add(&request->v, request->buf, request->size);
 380         block_acct_start(blk_get_stats(dataplane->blk), &request->acct,
 381                          request->v.size, BLOCK_ACCT_READ);
 382         request->aio_inflight++;
 383         blk_aio_preadv(dataplane->blk, request->start, &request->v, 0,
 384                        xen_block_complete_aio, request);
 385         break;
 386     case BLKIF_OP_WRITE:
 387     case BLKIF_OP_FLUSH_DISKCACHE:
 388         if (!request->req.nr_segments) {
 389             break;
 390         }
 391
 392         qemu_iovec_add(&request->v, request->buf, request->size);
 393         block_acct_start(blk_get_stats(dataplane->blk), &request->acct,
 394                          request->v.size,
 395                          request->req.operation == BLKIF_OP_WRITE ?
 396                          BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
 397         request->aio_inflight++;
 398         blk_aio_pwritev(dataplane->blk, request->start, &request->v, 0,
 399                         xen_block_complete_aio, request);
 400         break;
 401     case BLKIF_OP_DISCARD:
 402     {
 403         struct blkif_request_discard *req = (void *)&request->req;
 404         if (!xen_block_split_discard(request, req->sector_number,
 405                                      req->nr_sectors)) {
 406             goto err;
 407         }
 408         break;
 409     }
 410     default:
 411         /* unknown operation (shouldn't happen -- parse catches this) */
 412         goto err;
 413     }
 414
 415     xen_block_complete_aio(request, 0);
 416
 417     return 0;
 418
 419 err:
 420     xen_block_finish_request(request);
 421     request->status = BLKIF_RSP_ERROR;
 422     return -1;
 423 }
 424
 425 static int xen_block_send_response(XenBlockRequest *request)
 426 {
 427     XenBlockDataPlane *dataplane = request->dataplane;
 428     int send_notify = 0;
 429     int have_requests = 0;
 430     blkif_response_t *resp;
 431
 432     /* Place on the response ring for the relevant domain. */
 433     switch (dataplane->protocol) {
 434     case BLKIF_PROTOCOL_NATIVE:
 435         resp = (blkif_response_t *)RING_GET_RESPONSE(
 436             &dataplane->rings.native,
 437             dataplane->rings.native.rsp_prod_pvt);
 438         break;
 439     case BLKIF_PROTOCOL_X86_32:
 440         resp = (blkif_response_t *)RING_GET_RESPONSE(
 441             &dataplane->rings.x86_32_part,
 442             dataplane->rings.x86_32_part.rsp_prod_pvt);
 443         break;
 444     case BLKIF_PROTOCOL_X86_64:
 445         resp = (blkif_response_t *)RING_GET_RESPONSE(
 446             &dataplane->rings.x86_64_part,
 447             dataplane->rings.x86_64_part.rsp_prod_pvt);
 448         break;
 449     default:
 450         return 0;
 451     }
 452
 453     resp->id = request->req.id;
 454     resp->operation = request->req.operation;
 455     resp->status = request->status;
 456
 457     dataplane->rings.common.rsp_prod_pvt++;
 458
 459     RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&dataplane->rings.common,
 460                                          send_notify);
 461     if (dataplane->rings.common.rsp_prod_pvt ==
 462         dataplane->rings.common.req_cons) {
 463         /*
 464          * Tail check for pending requests. Allows frontend to avoid
 465          * notifications if requests are already in flight (lower
 466          * overheads and promotes batching).
 467          */
 468         RING_FINAL_CHECK_FOR_REQUESTS(&dataplane->rings.common,
 469                                       have_requests);
 470     } else if (RING_HAS_UNCONSUMED_REQUESTS(&dataplane->rings.common)) {
 471         have_requests = 1;
 472     }
 473
 474     if (have_requests) {
 475         dataplane->more_work++;
 476     }
 477     return send_notify;
 478 }
 479
 480 static int xen_block_get_request(XenBlockDataPlane *dataplane,
 481                                  XenBlockRequest *request, RING_IDX rc)
 482 {
 483     switch (dataplane->protocol) {
 484     case BLKIF_PROTOCOL_NATIVE: {
 485         blkif_request_t *req =
 486             RING_GET_REQUEST(&dataplane->rings.native, rc);
 487
 488         memcpy(&request->req, req, sizeof(request->req));
 489         break;
 490     }
 491     case BLKIF_PROTOCOL_X86_32: {
 492         blkif_x86_32_request_t *req =
 493             RING_GET_REQUEST(&dataplane->rings.x86_32_part, rc);
 494
 495         blkif_get_x86_32_req(&request->req, req);
 496         break;
 497     }
 498     case BLKIF_PROTOCOL_X86_64: {
 499         blkif_x86_64_request_t *req =
 500             RING_GET_REQUEST(&dataplane->rings.x86_64_part, rc);
 501
 502         blkif_get_x86_64_req(&request->req, req);
 503         break;
 504     }
 505     }
 506     /* Prevent the compiler from accessing the on-ring fields instead. */
 507     barrier();
 508     return 0;
 509 }
 510
 511 /*
 512  * Threshold of in-flight requests above which we will start using
 513  * blk_io_plug()/blk_io_unplug() to batch requests.
 514  */
 515 #define IO_PLUG_THRESHOLD 1
 516
 517 static void xen_block_handle_requests(XenBlockDataPlane *dataplane)
 518 {
 519     RING_IDX rc, rp;
 520     XenBlockRequest *request;
 521     int inflight_atstart = dataplane->requests_inflight;
 522     int batched = 0;
 523
 524     dataplane->more_work = 0;
 525
 526     rc = dataplane->rings.common.req_cons;
 527     rp = dataplane->rings.common.sring->req_prod;
 528     xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
 529
 530     /*
 531      * If there was more than IO_PLUG_THRESHOLD requests in flight
 532      * when we got here, this is an indication that there the bottleneck
 533      * is below us, so it's worth beginning to batch up I/O requests
 534      * rather than submitting them immediately. The maximum number
 535      * of requests we're willing to batch is the number already in
 536      * flight, so it can grow up to max_requests when the bottleneck
 537      * is below us.
 538      */
 539     if (inflight_atstart > IO_PLUG_THRESHOLD) {
 540         blk_io_plug(dataplane->blk);
 541     }
 542     while (rc != rp) {
 543         /* pull request from ring */
 544         if (RING_REQUEST_CONS_OVERFLOW(&dataplane->rings.common, rc)) {
 545             break;
 546         }
 547         request = xen_block_start_request(dataplane);
 548         if (request == NULL) {
 549             dataplane->more_work++;
 550             break;
 551         }
 552         xen_block_get_request(dataplane, request, rc);
 553         dataplane->rings.common.req_cons = ++rc;
 554
 555         /* parse them */
 556         if (xen_block_parse_request(request) != 0) {
 557             switch (request->req.operation) {
 558             case BLKIF_OP_READ:
 559                 block_acct_invalid(blk_get_stats(dataplane->blk),
 560                                    BLOCK_ACCT_READ);
 561                 break;
 562             case BLKIF_OP_WRITE:
 563                 block_acct_invalid(blk_get_stats(dataplane->blk),
 564                                    BLOCK_ACCT_WRITE);
 565                 break;
 566             case BLKIF_OP_FLUSH_DISKCACHE:
 567                 block_acct_invalid(blk_get_stats(dataplane->blk),
 568                                    BLOCK_ACCT_FLUSH);
 569             default:
 570                 break;
 571             };
 572
 573             if (xen_block_send_response(request)) {
 574                 Error *local_err = NULL;
 575
 576                 xen_device_notify_event_channel(dataplane->xendev,
 577                                                 dataplane->event_channel,
 578                                                 &local_err);
 579                 if (local_err) {
 580                     error_report_err(local_err);
 581                 }
 582             }
 583             xen_block_release_request(request);
 584             continue;
 585         }
 586
 587         if (inflight_atstart > IO_PLUG_THRESHOLD &&
 588             batched >= inflight_atstart) {
 589             blk_io_unplug(dataplane->blk);
 590         }
 591         xen_block_do_aio(request);
 592         if (inflight_atstart > IO_PLUG_THRESHOLD) {
 593             if (batched >= inflight_atstart) {
 594                 blk_io_plug(dataplane->blk);
 595                 batched = 0;
 596             } else {
 597                 batched++;
 598             }
 599         }
 600     }
 601     if (inflight_atstart > IO_PLUG_THRESHOLD) {
 602         blk_io_unplug(dataplane->blk);
 603     }
 604
 605     if (dataplane->more_work &&
 606         dataplane->requests_inflight < dataplane->max_requests) {
 607         qemu_bh_schedule(dataplane->bh);
 608     }
 609 }
 610
 611 static void xen_block_dataplane_bh(void *opaque)
 612 {
 613     XenBlockDataPlane *dataplane = opaque;
 614
 615     aio_context_acquire(dataplane->ctx);
 616     xen_block_handle_requests(dataplane);
 617     aio_context_release(dataplane->ctx);
 618 }
 619
 620 static void xen_block_dataplane_event(void *opaque)
 621 {
 622     XenBlockDataPlane *dataplane = opaque;
 623
 624     qemu_bh_schedule(dataplane->bh);
 625 }
 626
 627 XenBlockDataPlane *xen_block_dataplane_create(XenDevice *xendev,
 628                                               BlockConf *conf,
 629                                               IOThread *iothread)
 630 {
 631     XenBlockDataPlane *dataplane = g_new0(XenBlockDataPlane, 1);
 632
 633     dataplane->xendev = xendev;
 634     dataplane->blk = conf->blk;
 635
 636     QLIST_INIT(&dataplane->inflight);
 637     QLIST_INIT(&dataplane->freelist);
 638
 639     if (iothread) {
 640         dataplane->iothread = iothread;
 641         object_ref(OBJECT(dataplane->iothread));
 642         dataplane->ctx = iothread_get_aio_context(dataplane->iothread);
 643     } else {
 644         dataplane->ctx = qemu_get_aio_context();
 645     }
 646     dataplane->bh = aio_bh_new(dataplane->ctx, xen_block_dataplane_bh,
 647                                dataplane);
 648
 649     return dataplane;
 650 }
 651
 652 void xen_block_dataplane_destroy(XenBlockDataPlane *dataplane)
 653 {
 654     XenBlockRequest *request;
 655
 656     if (!dataplane) {
 657         return;
 658     }
 659
 660     while (!QLIST_EMPTY(&dataplane->freelist)) {
 661         request = QLIST_FIRST(&dataplane->freelist);
 662         QLIST_REMOVE(request, list);
 663         qemu_iovec_destroy(&request->v);
 664         qemu_vfree(request->buf);
 665         g_free(request);
 666     }
 667
 668     qemu_bh_delete(dataplane->bh);
 669     if (dataplane->iothread) {
 670         object_unref(OBJECT(dataplane->iothread));
 671     }
 672
 673     g_free(dataplane);
 674 }
 675
 676 void xen_block_dataplane_stop(XenBlockDataPlane *dataplane)
 677 {
 678     XenDevice *xendev;
 679
 680     if (!dataplane) {
 681         return;
 682     }
 683
 684     aio_context_acquire(dataplane->ctx);
 685     /* Xen doesn't have multiple users for nodes, so this can't fail */
 686     blk_set_aio_context(dataplane->blk, qemu_get_aio_context(), &error_abort);
 687     aio_context_release(dataplane->ctx);
 688
 689     xendev = dataplane->xendev;
 690
 691     if (dataplane->event_channel) {
 692         Error *local_err = NULL;
 693
 694         xen_device_unbind_event_channel(xendev, dataplane->event_channel,
 695                                         &local_err);
 696         dataplane->event_channel = NULL;
 697
 698         if (local_err) {
 699             error_report_err(local_err);
 700         }
 701     }
 702
 703     if (dataplane->sring) {
 704         Error *local_err = NULL;
 705
 706         xen_device_unmap_grant_refs(xendev, dataplane->sring,
 707                                     dataplane->nr_ring_ref, &local_err);
 708         dataplane->sring = NULL;
 709
 710         if (local_err) {
 711             error_report_err(local_err);
 712         }
 713     }
 714
 715     g_free(dataplane->ring_ref);
 716     dataplane->ring_ref = NULL;
 717 }
 718
 719 void xen_block_dataplane_start(XenBlockDataPlane *dataplane,
 720                                const unsigned int ring_ref[],
 721                                unsigned int nr_ring_ref,
 722                                unsigned int event_channel,
 723                                unsigned int protocol,
 724                                Error **errp)
 725 {
 726     XenDevice *xendev = dataplane->xendev;
 727     Error *local_err = NULL;
 728     unsigned int ring_size;
 729     unsigned int i;
 730
 731     dataplane->nr_ring_ref = nr_ring_ref;
 732     dataplane->ring_ref = g_new(unsigned int, nr_ring_ref);
 733
 734     for (i = 0; i < nr_ring_ref; i++) {
 735         dataplane->ring_ref[i] = ring_ref[i];
 736     }
 737
 738     dataplane->protocol = protocol;
 739
 740     ring_size = XC_PAGE_SIZE * dataplane->nr_ring_ref;
 741     switch (dataplane->protocol) {
 742     case BLKIF_PROTOCOL_NATIVE:
 743     {
 744         dataplane->max_requests = __CONST_RING_SIZE(blkif, ring_size);
 745         break;
 746     }
 747     case BLKIF_PROTOCOL_X86_32:
 748     {
 749         dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_32, ring_size);
 750         break;
 751     }
 752     case BLKIF_PROTOCOL_X86_64:
 753     {
 754         dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_64, ring_size);
 755         break;
 756     }
 757     default:
 758         error_setg(errp, "unknown protocol %u", dataplane->protocol);
 759         return;
 760     }
 761
 762     xen_device_set_max_grant_refs(xendev, dataplane->nr_ring_ref,
 763                                   &local_err);
 764     if (local_err) {
 765         error_propagate(errp, local_err);
 766         goto stop;
 767     }
 768
 769     dataplane->sring = xen_device_map_grant_refs(xendev,
 770                                               dataplane->ring_ref,
 771                                               dataplane->nr_ring_ref,
 772                                               PROT_READ | PROT_WRITE,
 773                                               &local_err);
 774     if (local_err) {
 775         error_propagate(errp, local_err);
 776         goto stop;
 777     }
 778
 779     switch (dataplane->protocol) {
 780     case BLKIF_PROTOCOL_NATIVE:
 781     {
 782         blkif_sring_t *sring_native = dataplane->sring;
 783
 784         BACK_RING_INIT(&dataplane->rings.native, sring_native, ring_size);
 785         break;
 786     }
 787     case BLKIF_PROTOCOL_X86_32:
 788     {
 789         blkif_x86_32_sring_t *sring_x86_32 = dataplane->sring;
 790
 791         BACK_RING_INIT(&dataplane->rings.x86_32_part, sring_x86_32,
 792                        ring_size);
 793         break;
 794     }
 795     case BLKIF_PROTOCOL_X86_64:
 796     {
 797         blkif_x86_64_sring_t *sring_x86_64 = dataplane->sring;
 798
 799         BACK_RING_INIT(&dataplane->rings.x86_64_part, sring_x86_64,
 800                        ring_size);
 801         break;
 802     }
 803     }
 804
 805     dataplane->event_channel =
 806         xen_device_bind_event_channel(xendev, event_channel,
 807                                       xen_block_dataplane_event, dataplane,
 808                                       &local_err);
 809     if (local_err) {
 810         error_propagate(errp, local_err);
 811         goto stop;
 812     }
 813
 814     aio_context_acquire(dataplane->ctx);
 815     /* If other users keep the BlockBackend in the iothread, that's ok */
 816     blk_set_aio_context(dataplane->blk, dataplane->ctx, NULL);
 817     aio_context_release(dataplane->ctx);
 818     return;
 819
 820 stop:
 821     xen_block_dataplane_stop(dataplane);
 822 }