]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (c) 2018 Citrix Systems Inc. | |
3 | * (c) Gerd Hoffmann <[email protected]> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License as published by | |
7 | * the Free Software Foundation; under version 2 of the License. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along | |
15 | * with this program; if not, see <http://www.gnu.org/licenses/>. | |
16 | * | |
17 | * Contributions after 2012-01-13 are licensed under the terms of the | |
18 | * GNU GPL, version 2 or (at your option) any later version. | |
19 | */ | |
20 | ||
21 | #include "qemu/osdep.h" | |
22 | #include "qemu/error-report.h" | |
23 | #include "qemu/main-loop.h" | |
24 | #include "qapi/error.h" | |
25 | #include "hw/xen/xen_common.h" | |
26 | #include "hw/block/xen_blkif.h" | |
27 | #include "sysemu/block-backend.h" | |
28 | #include "sysemu/iothread.h" | |
29 | #include "xen-block.h" | |
30 | ||
31 | typedef struct XenBlockRequest { | |
32 | blkif_request_t req; | |
33 | int16_t status; | |
34 | off_t start; | |
35 | QEMUIOVector v; | |
36 | void *buf; | |
37 | size_t size; | |
38 | int presync; | |
39 | int aio_inflight; | |
40 | int aio_errors; | |
41 | XenBlockDataPlane *dataplane; | |
42 | QLIST_ENTRY(XenBlockRequest) list; | |
43 | BlockAcctCookie acct; | |
44 | } XenBlockRequest; | |
45 | ||
46 | struct XenBlockDataPlane { | |
47 | XenDevice *xendev; | |
48 | XenEventChannel *event_channel; | |
49 | unsigned int *ring_ref; | |
50 | unsigned int nr_ring_ref; | |
51 | void *sring; | |
52 | int protocol; | |
53 | blkif_back_rings_t rings; | |
54 | int more_work; | |
55 | QLIST_HEAD(inflight_head, XenBlockRequest) inflight; | |
56 | QLIST_HEAD(freelist_head, XenBlockRequest) freelist; | |
57 | int requests_total; | |
58 | int requests_inflight; | |
59 | unsigned int max_requests; | |
60 | BlockBackend *blk; | |
61 | unsigned int sector_size; | |
62 | QEMUBH *bh; | |
63 | IOThread *iothread; | |
64 | AioContext *ctx; | |
65 | }; | |
66 | ||
67 | static void reset_request(XenBlockRequest *request) | |
68 | { | |
69 | memset(&request->req, 0, sizeof(request->req)); | |
70 | request->status = 0; | |
71 | request->start = 0; | |
72 | request->size = 0; | |
73 | request->presync = 0; | |
74 | ||
75 | request->aio_inflight = 0; | |
76 | request->aio_errors = 0; | |
77 | ||
78 | request->dataplane = NULL; | |
79 | memset(&request->list, 0, sizeof(request->list)); | |
80 | memset(&request->acct, 0, sizeof(request->acct)); | |
81 | ||
82 | qemu_iovec_reset(&request->v); | |
83 | } | |
84 | ||
85 | static XenBlockRequest *xen_block_start_request(XenBlockDataPlane *dataplane) | |
86 | { | |
87 | XenBlockRequest *request = NULL; | |
88 | ||
89 | if (QLIST_EMPTY(&dataplane->freelist)) { | |
90 | if (dataplane->requests_total >= dataplane->max_requests) { | |
91 | goto out; | |
92 | } | |
93 | /* allocate new struct */ | |
94 | request = g_malloc0(sizeof(*request)); | |
95 | request->dataplane = dataplane; | |
96 | /* | |
97 | * We cannot need more pages per requests than this, and since we | |
98 | * re-use requests, allocate the memory once here. It will be freed | |
99 | * xen_block_dataplane_destroy() when the request list is freed. | |
100 | */ | |
101 | request->buf = qemu_memalign(XC_PAGE_SIZE, | |
102 | BLKIF_MAX_SEGMENTS_PER_REQUEST * | |
103 | XC_PAGE_SIZE); | |
104 | dataplane->requests_total++; | |
105 | qemu_iovec_init(&request->v, 1); | |
106 | } else { | |
107 | /* get one from freelist */ | |
108 | request = QLIST_FIRST(&dataplane->freelist); | |
109 | QLIST_REMOVE(request, list); | |
110 | } | |
111 | QLIST_INSERT_HEAD(&dataplane->inflight, request, list); | |
112 | dataplane->requests_inflight++; | |
113 | ||
114 | out: | |
115 | return request; | |
116 | } | |
117 | ||
118 | static void xen_block_finish_request(XenBlockRequest *request) | |
119 | { | |
120 | XenBlockDataPlane *dataplane = request->dataplane; | |
121 | ||
122 | QLIST_REMOVE(request, list); | |
123 | dataplane->requests_inflight--; | |
124 | } | |
125 | ||
126 | static void xen_block_release_request(XenBlockRequest *request) | |
127 | { | |
128 | XenBlockDataPlane *dataplane = request->dataplane; | |
129 | ||
130 | QLIST_REMOVE(request, list); | |
131 | reset_request(request); | |
132 | request->dataplane = dataplane; | |
133 | QLIST_INSERT_HEAD(&dataplane->freelist, request, list); | |
134 | dataplane->requests_inflight--; | |
135 | } | |
136 | ||
137 | /* | |
138 | * translate request into iovec + start offset | |
139 | * do sanity checks along the way | |
140 | */ | |
141 | static int xen_block_parse_request(XenBlockRequest *request) | |
142 | { | |
143 | XenBlockDataPlane *dataplane = request->dataplane; | |
144 | size_t len; | |
145 | int i; | |
146 | ||
147 | switch (request->req.operation) { | |
148 | case BLKIF_OP_READ: | |
149 | break; | |
150 | case BLKIF_OP_FLUSH_DISKCACHE: | |
151 | request->presync = 1; | |
152 | if (!request->req.nr_segments) { | |
153 | return 0; | |
154 | } | |
155 | /* fall through */ | |
156 | case BLKIF_OP_WRITE: | |
157 | break; | |
158 | case BLKIF_OP_DISCARD: | |
159 | return 0; | |
160 | default: | |
161 | error_report("error: unknown operation (%d)", request->req.operation); | |
162 | goto err; | |
163 | }; | |
164 | ||
165 | if (request->req.operation != BLKIF_OP_READ && | |
166 | blk_is_read_only(dataplane->blk)) { | |
167 | error_report("error: write req for ro device"); | |
168 | goto err; | |
169 | } | |
170 | ||
171 | request->start = request->req.sector_number * dataplane->sector_size; | |
172 | for (i = 0; i < request->req.nr_segments; i++) { | |
173 | if (i == BLKIF_MAX_SEGMENTS_PER_REQUEST) { | |
174 | error_report("error: nr_segments too big"); | |
175 | goto err; | |
176 | } | |
177 | if (request->req.seg[i].first_sect > request->req.seg[i].last_sect) { | |
178 | error_report("error: first > last sector"); | |
179 | goto err; | |
180 | } | |
181 | if (request->req.seg[i].last_sect * dataplane->sector_size >= | |
182 | XC_PAGE_SIZE) { | |
183 | error_report("error: page crossing"); | |
184 | goto err; | |
185 | } | |
186 | ||
187 | len = (request->req.seg[i].last_sect - | |
188 | request->req.seg[i].first_sect + 1) * dataplane->sector_size; | |
189 | request->size += len; | |
190 | } | |
191 | if (request->start + request->size > blk_getlength(dataplane->blk)) { | |
192 | error_report("error: access beyond end of file"); | |
193 | goto err; | |
194 | } | |
195 | return 0; | |
196 | ||
197 | err: | |
198 | request->status = BLKIF_RSP_ERROR; | |
199 | return -1; | |
200 | } | |
201 | ||
202 | static int xen_block_copy_request(XenBlockRequest *request) | |
203 | { | |
204 | XenBlockDataPlane *dataplane = request->dataplane; | |
205 | XenDevice *xendev = dataplane->xendev; | |
206 | XenDeviceGrantCopySegment segs[BLKIF_MAX_SEGMENTS_PER_REQUEST]; | |
207 | int i, count; | |
208 | bool to_domain = (request->req.operation == BLKIF_OP_READ); | |
209 | void *virt = request->buf; | |
210 | Error *local_err = NULL; | |
211 | ||
212 | if (request->req.nr_segments == 0) { | |
213 | return 0; | |
214 | } | |
215 | ||
216 | count = request->req.nr_segments; | |
217 | ||
218 | for (i = 0; i < count; i++) { | |
219 | if (to_domain) { | |
220 | segs[i].dest.foreign.ref = request->req.seg[i].gref; | |
221 | segs[i].dest.foreign.offset = request->req.seg[i].first_sect * | |
222 | dataplane->sector_size; | |
223 | segs[i].source.virt = virt; | |
224 | } else { | |
225 | segs[i].source.foreign.ref = request->req.seg[i].gref; | |
226 | segs[i].source.foreign.offset = request->req.seg[i].first_sect * | |
227 | dataplane->sector_size; | |
228 | segs[i].dest.virt = virt; | |
229 | } | |
230 | segs[i].len = (request->req.seg[i].last_sect - | |
231 | request->req.seg[i].first_sect + 1) * | |
232 | dataplane->sector_size; | |
233 | virt += segs[i].len; | |
234 | } | |
235 | ||
236 | xen_device_copy_grant_refs(xendev, to_domain, segs, count, &local_err); | |
237 | ||
238 | if (local_err) { | |
239 | error_reportf_err(local_err, "failed to copy data: "); | |
240 | ||
241 | request->aio_errors++; | |
242 | return -1; | |
243 | } | |
244 | ||
245 | return 0; | |
246 | } | |
247 | ||
248 | static int xen_block_do_aio(XenBlockRequest *request); | |
249 | static int xen_block_send_response(XenBlockRequest *request); | |
250 | ||
251 | static void xen_block_complete_aio(void *opaque, int ret) | |
252 | { | |
253 | XenBlockRequest *request = opaque; | |
254 | XenBlockDataPlane *dataplane = request->dataplane; | |
255 | ||
256 | aio_context_acquire(dataplane->ctx); | |
257 | ||
258 | if (ret != 0) { | |
259 | error_report("%s I/O error", | |
260 | request->req.operation == BLKIF_OP_READ ? | |
261 | "read" : "write"); | |
262 | request->aio_errors++; | |
263 | } | |
264 | ||
265 | request->aio_inflight--; | |
266 | if (request->presync) { | |
267 | request->presync = 0; | |
268 | xen_block_do_aio(request); | |
269 | goto done; | |
270 | } | |
271 | if (request->aio_inflight > 0) { | |
272 | goto done; | |
273 | } | |
274 | ||
275 | switch (request->req.operation) { | |
276 | case BLKIF_OP_READ: | |
277 | /* in case of failure request->aio_errors is increased */ | |
278 | if (ret == 0) { | |
279 | xen_block_copy_request(request); | |
280 | } | |
281 | break; | |
282 | case BLKIF_OP_WRITE: | |
283 | case BLKIF_OP_FLUSH_DISKCACHE: | |
284 | default: | |
285 | break; | |
286 | } | |
287 | ||
288 | request->status = request->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY; | |
289 | xen_block_finish_request(request); | |
290 | ||
291 | switch (request->req.operation) { | |
292 | case BLKIF_OP_WRITE: | |
293 | case BLKIF_OP_FLUSH_DISKCACHE: | |
294 | if (!request->req.nr_segments) { | |
295 | break; | |
296 | } | |
297 | /* fall through */ | |
298 | case BLKIF_OP_READ: | |
299 | if (request->status == BLKIF_RSP_OKAY) { | |
300 | block_acct_done(blk_get_stats(dataplane->blk), &request->acct); | |
301 | } else { | |
302 | block_acct_failed(blk_get_stats(dataplane->blk), &request->acct); | |
303 | } | |
304 | break; | |
305 | case BLKIF_OP_DISCARD: | |
306 | default: | |
307 | break; | |
308 | } | |
309 | if (xen_block_send_response(request)) { | |
310 | Error *local_err = NULL; | |
311 | ||
312 | xen_device_notify_event_channel(dataplane->xendev, | |
313 | dataplane->event_channel, | |
314 | &local_err); | |
315 | if (local_err) { | |
316 | error_report_err(local_err); | |
317 | } | |
318 | } | |
319 | xen_block_release_request(request); | |
320 | ||
321 | if (dataplane->more_work) { | |
322 | qemu_bh_schedule(dataplane->bh); | |
323 | } | |
324 | ||
325 | done: | |
326 | aio_context_release(dataplane->ctx); | |
327 | } | |
328 | ||
329 | static bool xen_block_split_discard(XenBlockRequest *request, | |
330 | blkif_sector_t sector_number, | |
331 | uint64_t nr_sectors) | |
332 | { | |
333 | XenBlockDataPlane *dataplane = request->dataplane; | |
334 | int64_t byte_offset; | |
335 | int byte_chunk; | |
336 | uint64_t byte_remaining; | |
337 | uint64_t sec_start = sector_number; | |
338 | uint64_t sec_count = nr_sectors; | |
339 | ||
340 | /* Wrap around, or overflowing byte limit? */ | |
341 | if (sec_start + sec_count < sec_count || | |
342 | sec_start + sec_count > INT64_MAX / dataplane->sector_size) { | |
343 | return false; | |
344 | } | |
345 | ||
346 | byte_offset = sec_start * dataplane->sector_size; | |
347 | byte_remaining = sec_count * dataplane->sector_size; | |
348 | ||
349 | do { | |
350 | byte_chunk = byte_remaining > BDRV_REQUEST_MAX_BYTES ? | |
351 | BDRV_REQUEST_MAX_BYTES : byte_remaining; | |
352 | request->aio_inflight++; | |
353 | blk_aio_pdiscard(dataplane->blk, byte_offset, byte_chunk, | |
354 | xen_block_complete_aio, request); | |
355 | byte_remaining -= byte_chunk; | |
356 | byte_offset += byte_chunk; | |
357 | } while (byte_remaining > 0); | |
358 | ||
359 | return true; | |
360 | } | |
361 | ||
362 | static int xen_block_do_aio(XenBlockRequest *request) | |
363 | { | |
364 | XenBlockDataPlane *dataplane = request->dataplane; | |
365 | ||
366 | if (request->req.nr_segments && | |
367 | (request->req.operation == BLKIF_OP_WRITE || | |
368 | request->req.operation == BLKIF_OP_FLUSH_DISKCACHE) && | |
369 | xen_block_copy_request(request)) { | |
370 | goto err; | |
371 | } | |
372 | ||
373 | request->aio_inflight++; | |
374 | if (request->presync) { | |
375 | blk_aio_flush(request->dataplane->blk, xen_block_complete_aio, | |
376 | request); | |
377 | return 0; | |
378 | } | |
379 | ||
380 | switch (request->req.operation) { | |
381 | case BLKIF_OP_READ: | |
382 | qemu_iovec_add(&request->v, request->buf, request->size); | |
383 | block_acct_start(blk_get_stats(dataplane->blk), &request->acct, | |
384 | request->v.size, BLOCK_ACCT_READ); | |
385 | request->aio_inflight++; | |
386 | blk_aio_preadv(dataplane->blk, request->start, &request->v, 0, | |
387 | xen_block_complete_aio, request); | |
388 | break; | |
389 | case BLKIF_OP_WRITE: | |
390 | case BLKIF_OP_FLUSH_DISKCACHE: | |
391 | if (!request->req.nr_segments) { | |
392 | break; | |
393 | } | |
394 | ||
395 | qemu_iovec_add(&request->v, request->buf, request->size); | |
396 | block_acct_start(blk_get_stats(dataplane->blk), &request->acct, | |
397 | request->v.size, | |
398 | request->req.operation == BLKIF_OP_WRITE ? | |
399 | BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH); | |
400 | request->aio_inflight++; | |
401 | blk_aio_pwritev(dataplane->blk, request->start, &request->v, 0, | |
402 | xen_block_complete_aio, request); | |
403 | break; | |
404 | case BLKIF_OP_DISCARD: | |
405 | { | |
406 | struct blkif_request_discard *req = (void *)&request->req; | |
407 | if (!xen_block_split_discard(request, req->sector_number, | |
408 | req->nr_sectors)) { | |
409 | goto err; | |
410 | } | |
411 | break; | |
412 | } | |
413 | default: | |
414 | /* unknown operation (shouldn't happen -- parse catches this) */ | |
415 | goto err; | |
416 | } | |
417 | ||
418 | xen_block_complete_aio(request, 0); | |
419 | ||
420 | return 0; | |
421 | ||
422 | err: | |
423 | xen_block_finish_request(request); | |
424 | request->status = BLKIF_RSP_ERROR; | |
425 | return -1; | |
426 | } | |
427 | ||
428 | static int xen_block_send_response(XenBlockRequest *request) | |
429 | { | |
430 | XenBlockDataPlane *dataplane = request->dataplane; | |
431 | int send_notify = 0; | |
432 | int have_requests = 0; | |
433 | blkif_response_t *resp; | |
434 | ||
435 | /* Place on the response ring for the relevant domain. */ | |
436 | switch (dataplane->protocol) { | |
437 | case BLKIF_PROTOCOL_NATIVE: | |
438 | resp = (blkif_response_t *)RING_GET_RESPONSE( | |
439 | &dataplane->rings.native, | |
440 | dataplane->rings.native.rsp_prod_pvt); | |
441 | break; | |
442 | case BLKIF_PROTOCOL_X86_32: | |
443 | resp = (blkif_response_t *)RING_GET_RESPONSE( | |
444 | &dataplane->rings.x86_32_part, | |
445 | dataplane->rings.x86_32_part.rsp_prod_pvt); | |
446 | break; | |
447 | case BLKIF_PROTOCOL_X86_64: | |
448 | resp = (blkif_response_t *)RING_GET_RESPONSE( | |
449 | &dataplane->rings.x86_64_part, | |
450 | dataplane->rings.x86_64_part.rsp_prod_pvt); | |
451 | break; | |
452 | default: | |
453 | return 0; | |
454 | } | |
455 | ||
456 | resp->id = request->req.id; | |
457 | resp->operation = request->req.operation; | |
458 | resp->status = request->status; | |
459 | ||
460 | dataplane->rings.common.rsp_prod_pvt++; | |
461 | ||
462 | RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&dataplane->rings.common, | |
463 | send_notify); | |
464 | if (dataplane->rings.common.rsp_prod_pvt == | |
465 | dataplane->rings.common.req_cons) { | |
466 | /* | |
467 | * Tail check for pending requests. Allows frontend to avoid | |
468 | * notifications if requests are already in flight (lower | |
469 | * overheads and promotes batching). | |
470 | */ | |
471 | RING_FINAL_CHECK_FOR_REQUESTS(&dataplane->rings.common, | |
472 | have_requests); | |
473 | } else if (RING_HAS_UNCONSUMED_REQUESTS(&dataplane->rings.common)) { | |
474 | have_requests = 1; | |
475 | } | |
476 | ||
477 | if (have_requests) { | |
478 | dataplane->more_work++; | |
479 | } | |
480 | return send_notify; | |
481 | } | |
482 | ||
483 | static int xen_block_get_request(XenBlockDataPlane *dataplane, | |
484 | XenBlockRequest *request, RING_IDX rc) | |
485 | { | |
486 | switch (dataplane->protocol) { | |
487 | case BLKIF_PROTOCOL_NATIVE: { | |
488 | blkif_request_t *req = | |
489 | RING_GET_REQUEST(&dataplane->rings.native, rc); | |
490 | ||
491 | memcpy(&request->req, req, sizeof(request->req)); | |
492 | break; | |
493 | } | |
494 | case BLKIF_PROTOCOL_X86_32: { | |
495 | blkif_x86_32_request_t *req = | |
496 | RING_GET_REQUEST(&dataplane->rings.x86_32_part, rc); | |
497 | ||
498 | blkif_get_x86_32_req(&request->req, req); | |
499 | break; | |
500 | } | |
501 | case BLKIF_PROTOCOL_X86_64: { | |
502 | blkif_x86_64_request_t *req = | |
503 | RING_GET_REQUEST(&dataplane->rings.x86_64_part, rc); | |
504 | ||
505 | blkif_get_x86_64_req(&request->req, req); | |
506 | break; | |
507 | } | |
508 | } | |
509 | /* Prevent the compiler from accessing the on-ring fields instead. */ | |
510 | barrier(); | |
511 | return 0; | |
512 | } | |
513 | ||
514 | /* | |
515 | * Threshold of in-flight requests above which we will start using | |
516 | * blk_io_plug()/blk_io_unplug() to batch requests. | |
517 | */ | |
518 | #define IO_PLUG_THRESHOLD 1 | |
519 | ||
520 | static bool xen_block_handle_requests(XenBlockDataPlane *dataplane) | |
521 | { | |
522 | RING_IDX rc, rp; | |
523 | XenBlockRequest *request; | |
524 | int inflight_atstart = dataplane->requests_inflight; | |
525 | int batched = 0; | |
526 | bool done_something = false; | |
527 | ||
528 | dataplane->more_work = 0; | |
529 | ||
530 | rc = dataplane->rings.common.req_cons; | |
531 | rp = dataplane->rings.common.sring->req_prod; | |
532 | xen_rmb(); /* Ensure we see queued requests up to 'rp'. */ | |
533 | ||
534 | /* | |
535 | * If there was more than IO_PLUG_THRESHOLD requests in flight | |
536 | * when we got here, this is an indication that there the bottleneck | |
537 | * is below us, so it's worth beginning to batch up I/O requests | |
538 | * rather than submitting them immediately. The maximum number | |
539 | * of requests we're willing to batch is the number already in | |
540 | * flight, so it can grow up to max_requests when the bottleneck | |
541 | * is below us. | |
542 | */ | |
543 | if (inflight_atstart > IO_PLUG_THRESHOLD) { | |
544 | blk_io_plug(dataplane->blk); | |
545 | } | |
546 | while (rc != rp) { | |
547 | /* pull request from ring */ | |
548 | if (RING_REQUEST_CONS_OVERFLOW(&dataplane->rings.common, rc)) { | |
549 | break; | |
550 | } | |
551 | request = xen_block_start_request(dataplane); | |
552 | if (request == NULL) { | |
553 | dataplane->more_work++; | |
554 | break; | |
555 | } | |
556 | xen_block_get_request(dataplane, request, rc); | |
557 | dataplane->rings.common.req_cons = ++rc; | |
558 | done_something = true; | |
559 | ||
560 | /* parse them */ | |
561 | if (xen_block_parse_request(request) != 0) { | |
562 | switch (request->req.operation) { | |
563 | case BLKIF_OP_READ: | |
564 | block_acct_invalid(blk_get_stats(dataplane->blk), | |
565 | BLOCK_ACCT_READ); | |
566 | break; | |
567 | case BLKIF_OP_WRITE: | |
568 | block_acct_invalid(blk_get_stats(dataplane->blk), | |
569 | BLOCK_ACCT_WRITE); | |
570 | break; | |
571 | case BLKIF_OP_FLUSH_DISKCACHE: | |
572 | block_acct_invalid(blk_get_stats(dataplane->blk), | |
573 | BLOCK_ACCT_FLUSH); | |
574 | default: | |
575 | break; | |
576 | }; | |
577 | ||
578 | if (xen_block_send_response(request)) { | |
579 | Error *local_err = NULL; | |
580 | ||
581 | xen_device_notify_event_channel(dataplane->xendev, | |
582 | dataplane->event_channel, | |
583 | &local_err); | |
584 | if (local_err) { | |
585 | error_report_err(local_err); | |
586 | } | |
587 | } | |
588 | xen_block_release_request(request); | |
589 | continue; | |
590 | } | |
591 | ||
592 | if (inflight_atstart > IO_PLUG_THRESHOLD && | |
593 | batched >= inflight_atstart) { | |
594 | blk_io_unplug(dataplane->blk); | |
595 | } | |
596 | xen_block_do_aio(request); | |
597 | if (inflight_atstart > IO_PLUG_THRESHOLD) { | |
598 | if (batched >= inflight_atstart) { | |
599 | blk_io_plug(dataplane->blk); | |
600 | batched = 0; | |
601 | } else { | |
602 | batched++; | |
603 | } | |
604 | } | |
605 | } | |
606 | if (inflight_atstart > IO_PLUG_THRESHOLD) { | |
607 | blk_io_unplug(dataplane->blk); | |
608 | } | |
609 | ||
610 | return done_something; | |
611 | } | |
612 | ||
613 | static void xen_block_dataplane_bh(void *opaque) | |
614 | { | |
615 | XenBlockDataPlane *dataplane = opaque; | |
616 | ||
617 | aio_context_acquire(dataplane->ctx); | |
618 | xen_block_handle_requests(dataplane); | |
619 | aio_context_release(dataplane->ctx); | |
620 | } | |
621 | ||
622 | static bool xen_block_dataplane_event(void *opaque) | |
623 | { | |
624 | XenBlockDataPlane *dataplane = opaque; | |
625 | ||
626 | return xen_block_handle_requests(dataplane); | |
627 | } | |
628 | ||
629 | XenBlockDataPlane *xen_block_dataplane_create(XenDevice *xendev, | |
630 | BlockBackend *blk, | |
631 | unsigned int sector_size, | |
632 | IOThread *iothread) | |
633 | { | |
634 | XenBlockDataPlane *dataplane = g_new0(XenBlockDataPlane, 1); | |
635 | ||
636 | dataplane->xendev = xendev; | |
637 | dataplane->blk = blk; | |
638 | dataplane->sector_size = sector_size; | |
639 | ||
640 | QLIST_INIT(&dataplane->inflight); | |
641 | QLIST_INIT(&dataplane->freelist); | |
642 | ||
643 | if (iothread) { | |
644 | dataplane->iothread = iothread; | |
645 | object_ref(OBJECT(dataplane->iothread)); | |
646 | dataplane->ctx = iothread_get_aio_context(dataplane->iothread); | |
647 | } else { | |
648 | dataplane->ctx = qemu_get_aio_context(); | |
649 | } | |
650 | dataplane->bh = aio_bh_new(dataplane->ctx, xen_block_dataplane_bh, | |
651 | dataplane); | |
652 | ||
653 | return dataplane; | |
654 | } | |
655 | ||
656 | void xen_block_dataplane_destroy(XenBlockDataPlane *dataplane) | |
657 | { | |
658 | XenBlockRequest *request; | |
659 | ||
660 | if (!dataplane) { | |
661 | return; | |
662 | } | |
663 | ||
664 | while (!QLIST_EMPTY(&dataplane->freelist)) { | |
665 | request = QLIST_FIRST(&dataplane->freelist); | |
666 | QLIST_REMOVE(request, list); | |
667 | qemu_iovec_destroy(&request->v); | |
668 | qemu_vfree(request->buf); | |
669 | g_free(request); | |
670 | } | |
671 | ||
672 | qemu_bh_delete(dataplane->bh); | |
673 | if (dataplane->iothread) { | |
674 | object_unref(OBJECT(dataplane->iothread)); | |
675 | } | |
676 | ||
677 | g_free(dataplane); | |
678 | } | |
679 | ||
680 | void xen_block_dataplane_stop(XenBlockDataPlane *dataplane) | |
681 | { | |
682 | XenDevice *xendev; | |
683 | ||
684 | if (!dataplane) { | |
685 | return; | |
686 | } | |
687 | ||
688 | aio_context_acquire(dataplane->ctx); | |
689 | /* Xen doesn't have multiple users for nodes, so this can't fail */ | |
690 | blk_set_aio_context(dataplane->blk, qemu_get_aio_context(), &error_abort); | |
691 | aio_context_release(dataplane->ctx); | |
692 | ||
693 | xendev = dataplane->xendev; | |
694 | ||
695 | if (dataplane->event_channel) { | |
696 | Error *local_err = NULL; | |
697 | ||
698 | xen_device_unbind_event_channel(xendev, dataplane->event_channel, | |
699 | &local_err); | |
700 | dataplane->event_channel = NULL; | |
701 | ||
702 | if (local_err) { | |
703 | error_report_err(local_err); | |
704 | } | |
705 | } | |
706 | ||
707 | if (dataplane->sring) { | |
708 | Error *local_err = NULL; | |
709 | ||
710 | xen_device_unmap_grant_refs(xendev, dataplane->sring, | |
711 | dataplane->nr_ring_ref, &local_err); | |
712 | dataplane->sring = NULL; | |
713 | ||
714 | if (local_err) { | |
715 | error_report_err(local_err); | |
716 | } | |
717 | } | |
718 | ||
719 | g_free(dataplane->ring_ref); | |
720 | dataplane->ring_ref = NULL; | |
721 | } | |
722 | ||
723 | void xen_block_dataplane_start(XenBlockDataPlane *dataplane, | |
724 | const unsigned int ring_ref[], | |
725 | unsigned int nr_ring_ref, | |
726 | unsigned int event_channel, | |
727 | unsigned int protocol, | |
728 | Error **errp) | |
729 | { | |
730 | XenDevice *xendev = dataplane->xendev; | |
731 | Error *local_err = NULL; | |
732 | unsigned int ring_size; | |
733 | unsigned int i; | |
734 | ||
735 | dataplane->nr_ring_ref = nr_ring_ref; | |
736 | dataplane->ring_ref = g_new(unsigned int, nr_ring_ref); | |
737 | ||
738 | for (i = 0; i < nr_ring_ref; i++) { | |
739 | dataplane->ring_ref[i] = ring_ref[i]; | |
740 | } | |
741 | ||
742 | dataplane->protocol = protocol; | |
743 | ||
744 | ring_size = XC_PAGE_SIZE * dataplane->nr_ring_ref; | |
745 | switch (dataplane->protocol) { | |
746 | case BLKIF_PROTOCOL_NATIVE: | |
747 | { | |
748 | dataplane->max_requests = __CONST_RING_SIZE(blkif, ring_size); | |
749 | break; | |
750 | } | |
751 | case BLKIF_PROTOCOL_X86_32: | |
752 | { | |
753 | dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_32, ring_size); | |
754 | break; | |
755 | } | |
756 | case BLKIF_PROTOCOL_X86_64: | |
757 | { | |
758 | dataplane->max_requests = __CONST_RING_SIZE(blkif_x86_64, ring_size); | |
759 | break; | |
760 | } | |
761 | default: | |
762 | error_setg(errp, "unknown protocol %u", dataplane->protocol); | |
763 | return; | |
764 | } | |
765 | ||
766 | xen_device_set_max_grant_refs(xendev, dataplane->nr_ring_ref, | |
767 | &local_err); | |
768 | if (local_err) { | |
769 | error_propagate(errp, local_err); | |
770 | goto stop; | |
771 | } | |
772 | ||
773 | dataplane->sring = xen_device_map_grant_refs(xendev, | |
774 | dataplane->ring_ref, | |
775 | dataplane->nr_ring_ref, | |
776 | PROT_READ | PROT_WRITE, | |
777 | &local_err); | |
778 | if (local_err) { | |
779 | error_propagate(errp, local_err); | |
780 | goto stop; | |
781 | } | |
782 | ||
783 | switch (dataplane->protocol) { | |
784 | case BLKIF_PROTOCOL_NATIVE: | |
785 | { | |
786 | blkif_sring_t *sring_native = dataplane->sring; | |
787 | ||
788 | BACK_RING_INIT(&dataplane->rings.native, sring_native, ring_size); | |
789 | break; | |
790 | } | |
791 | case BLKIF_PROTOCOL_X86_32: | |
792 | { | |
793 | blkif_x86_32_sring_t *sring_x86_32 = dataplane->sring; | |
794 | ||
795 | BACK_RING_INIT(&dataplane->rings.x86_32_part, sring_x86_32, | |
796 | ring_size); | |
797 | break; | |
798 | } | |
799 | case BLKIF_PROTOCOL_X86_64: | |
800 | { | |
801 | blkif_x86_64_sring_t *sring_x86_64 = dataplane->sring; | |
802 | ||
803 | BACK_RING_INIT(&dataplane->rings.x86_64_part, sring_x86_64, | |
804 | ring_size); | |
805 | break; | |
806 | } | |
807 | } | |
808 | ||
809 | dataplane->event_channel = | |
810 | xen_device_bind_event_channel(xendev, dataplane->ctx, event_channel, | |
811 | xen_block_dataplane_event, dataplane, | |
812 | &local_err); | |
813 | if (local_err) { | |
814 | error_propagate(errp, local_err); | |
815 | goto stop; | |
816 | } | |
817 | ||
818 | aio_context_acquire(dataplane->ctx); | |
819 | /* If other users keep the BlockBackend in the iothread, that's ok */ | |
820 | blk_set_aio_context(dataplane->blk, dataplane->ctx, NULL); | |
821 | aio_context_release(dataplane->ctx); | |
822 | return; | |
823 | ||
824 | stop: | |
825 | xen_block_dataplane_stop(dataplane); | |
826 | } |