]>
Commit | Line | Data |
---|---|---|
f27aaf4b CB |
1 | /* |
2 | * QEMU Block driver for RADOS (Ceph) | |
3 | * | |
ad32e9c0 JD |
4 | * Copyright (C) 2010-2011 Christian Brunner <[email protected]>, |
5 | * Josh Durgin <[email protected]> | |
f27aaf4b CB |
6 | * |
7 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
8 | * the COPYING file in the top-level directory. | |
9 | * | |
6b620ca3 PB |
10 | * Contributions after 2012-01-13 are licensed under the terms of the |
11 | * GNU GPL, version 2 or (at your option) any later version. | |
f27aaf4b CB |
12 | */ |
13 | ||
ad32e9c0 JD |
14 | #include <inttypes.h> |
15 | ||
f27aaf4b | 16 | #include "qemu-common.h" |
1de7afc9 | 17 | #include "qemu/error-report.h" |
737e150e | 18 | #include "block/block_int.h" |
f27aaf4b | 19 | |
ad32e9c0 | 20 | #include <rbd/librbd.h> |
f27aaf4b | 21 | |
f27aaf4b CB |
22 | /* |
23 | * When specifying the image filename use: | |
24 | * | |
fab5cf59 | 25 | * rbd:poolname/devicename[@snapshotname][:option1=value1[:option2=value2...]] |
f27aaf4b | 26 | * |
9e1fbcde | 27 | * poolname must be the name of an existing rados pool. |
f27aaf4b | 28 | * |
9e1fbcde | 29 | * devicename is the name of the rbd image. |
f27aaf4b | 30 | * |
9e1fbcde SW |
31 | * Each option given is used to configure rados, and may be any valid |
32 | * Ceph option, "id", or "conf". | |
fab5cf59 | 33 | * |
9e1fbcde SW |
34 | * The "id" option indicates what user we should authenticate as to |
35 | * the Ceph cluster. If it is excluded we will use the Ceph default | |
36 | * (normally 'admin'). | |
f27aaf4b | 37 | * |
9e1fbcde SW |
38 | * The "conf" option specifies a Ceph configuration file to read. If |
39 | * it is not specified, we will read from the default Ceph locations | |
40 | * (e.g., /etc/ceph/ceph.conf). To avoid reading _any_ configuration | |
41 | * file, specify conf=/dev/null. | |
f27aaf4b | 42 | * |
9e1fbcde SW |
43 | * Configuration values containing :, @, or = can be escaped with a |
44 | * leading "\". | |
f27aaf4b CB |
45 | */ |
46 | ||
787f3133 JD |
47 | /* rbd_aio_discard added in 0.1.2 */ |
48 | #if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2) | |
49 | #define LIBRBD_SUPPORTS_DISCARD | |
50 | #else | |
51 | #undef LIBRBD_SUPPORTS_DISCARD | |
52 | #endif | |
53 | ||
f27aaf4b CB |
54 | #define OBJ_MAX_SIZE (1UL << OBJ_DEFAULT_OBJ_ORDER) |
55 | ||
ad32e9c0 JD |
56 | #define RBD_MAX_CONF_NAME_SIZE 128 |
57 | #define RBD_MAX_CONF_VAL_SIZE 512 | |
58 | #define RBD_MAX_CONF_SIZE 1024 | |
59 | #define RBD_MAX_POOL_NAME_SIZE 128 | |
60 | #define RBD_MAX_SNAP_NAME_SIZE 128 | |
61 | #define RBD_MAX_SNAPS 100 | |
62 | ||
787f3133 JD |
63 | typedef enum { |
64 | RBD_AIO_READ, | |
65 | RBD_AIO_WRITE, | |
dc7588c1 JD |
66 | RBD_AIO_DISCARD, |
67 | RBD_AIO_FLUSH | |
787f3133 JD |
68 | } RBDAIOCmd; |
69 | ||
f27aaf4b CB |
70 | typedef struct RBDAIOCB { |
71 | BlockDriverAIOCB common; | |
72 | QEMUBH *bh; | |
08448d51 | 73 | int64_t ret; |
f27aaf4b CB |
74 | QEMUIOVector *qiov; |
75 | char *bounce; | |
787f3133 | 76 | RBDAIOCmd cmd; |
f27aaf4b | 77 | int64_t sector_num; |
f27aaf4b CB |
78 | int error; |
79 | struct BDRVRBDState *s; | |
80 | int cancelled; | |
473c7f02 | 81 | int status; |
f27aaf4b CB |
82 | } RBDAIOCB; |
83 | ||
84 | typedef struct RADOSCB { | |
85 | int rcbid; | |
86 | RBDAIOCB *acb; | |
87 | struct BDRVRBDState *s; | |
88 | int done; | |
ad32e9c0 | 89 | int64_t size; |
f27aaf4b | 90 | char *buf; |
08448d51 | 91 | int64_t ret; |
f27aaf4b CB |
92 | } RADOSCB; |
93 | ||
94 | #define RBD_FD_READ 0 | |
95 | #define RBD_FD_WRITE 1 | |
96 | ||
97 | typedef struct BDRVRBDState { | |
98 | int fds[2]; | |
ad32e9c0 JD |
99 | rados_t cluster; |
100 | rados_ioctx_t io_ctx; | |
101 | rbd_image_t image; | |
102 | char name[RBD_MAX_IMAGE_NAME_SIZE]; | |
ad32e9c0 | 103 | char *snap; |
f27aaf4b CB |
104 | int event_reader_pos; |
105 | RADOSCB *event_rcb; | |
106 | } BDRVRBDState; | |
107 | ||
f27aaf4b CB |
108 | static void rbd_aio_bh_cb(void *opaque); |
109 | ||
ad32e9c0 JD |
110 | static int qemu_rbd_next_tok(char *dst, int dst_len, |
111 | char *src, char delim, | |
112 | const char *name, | |
113 | char **p) | |
f27aaf4b CB |
114 | { |
115 | int l; | |
116 | char *end; | |
117 | ||
118 | *p = NULL; | |
119 | ||
120 | if (delim != '\0') { | |
16a06b24 SW |
121 | for (end = src; *end; ++end) { |
122 | if (*end == delim) { | |
123 | break; | |
124 | } | |
125 | if (*end == '\\' && end[1] != '\0') { | |
126 | end++; | |
127 | } | |
128 | } | |
129 | if (*end == delim) { | |
f27aaf4b CB |
130 | *p = end + 1; |
131 | *end = '\0'; | |
132 | } | |
133 | } | |
134 | l = strlen(src); | |
135 | if (l >= dst_len) { | |
136 | error_report("%s too long", name); | |
137 | return -EINVAL; | |
138 | } else if (l == 0) { | |
139 | error_report("%s too short", name); | |
140 | return -EINVAL; | |
141 | } | |
142 | ||
143 | pstrcpy(dst, dst_len, src); | |
144 | ||
145 | return 0; | |
146 | } | |
147 | ||
16a06b24 SW |
148 | static void qemu_rbd_unescape(char *src) |
149 | { | |
150 | char *p; | |
151 | ||
152 | for (p = src; *src; ++src, ++p) { | |
153 | if (*src == '\\' && src[1] != '\0') { | |
154 | src++; | |
155 | } | |
156 | *p = *src; | |
157 | } | |
158 | *p = '\0'; | |
159 | } | |
160 | ||
ad32e9c0 JD |
161 | static int qemu_rbd_parsename(const char *filename, |
162 | char *pool, int pool_len, | |
163 | char *snap, int snap_len, | |
fab5cf59 JD |
164 | char *name, int name_len, |
165 | char *conf, int conf_len) | |
f27aaf4b CB |
166 | { |
167 | const char *start; | |
168 | char *p, *buf; | |
169 | int ret; | |
170 | ||
171 | if (!strstart(filename, "rbd:", &start)) { | |
172 | return -EINVAL; | |
173 | } | |
174 | ||
7267c094 | 175 | buf = g_strdup(start); |
f27aaf4b | 176 | p = buf; |
fab5cf59 JD |
177 | *snap = '\0'; |
178 | *conf = '\0'; | |
f27aaf4b | 179 | |
ad32e9c0 | 180 | ret = qemu_rbd_next_tok(pool, pool_len, p, '/', "pool name", &p); |
f27aaf4b CB |
181 | if (ret < 0 || !p) { |
182 | ret = -EINVAL; | |
183 | goto done; | |
184 | } | |
16a06b24 | 185 | qemu_rbd_unescape(pool); |
fab5cf59 JD |
186 | |
187 | if (strchr(p, '@')) { | |
188 | ret = qemu_rbd_next_tok(name, name_len, p, '@', "object name", &p); | |
189 | if (ret < 0) { | |
190 | goto done; | |
191 | } | |
192 | ret = qemu_rbd_next_tok(snap, snap_len, p, ':', "snap name", &p); | |
16a06b24 | 193 | qemu_rbd_unescape(snap); |
fab5cf59 JD |
194 | } else { |
195 | ret = qemu_rbd_next_tok(name, name_len, p, ':', "object name", &p); | |
f27aaf4b | 196 | } |
16a06b24 | 197 | qemu_rbd_unescape(name); |
fab5cf59 | 198 | if (ret < 0 || !p) { |
f27aaf4b CB |
199 | goto done; |
200 | } | |
201 | ||
fab5cf59 | 202 | ret = qemu_rbd_next_tok(conf, conf_len, p, '\0', "configuration", &p); |
f27aaf4b CB |
203 | |
204 | done: | |
7267c094 | 205 | g_free(buf); |
f27aaf4b CB |
206 | return ret; |
207 | } | |
208 | ||
7c7e9df0 SW |
209 | static char *qemu_rbd_parse_clientname(const char *conf, char *clientname) |
210 | { | |
211 | const char *p = conf; | |
212 | ||
213 | while (*p) { | |
214 | int len; | |
215 | const char *end = strchr(p, ':'); | |
216 | ||
217 | if (end) { | |
218 | len = end - p; | |
219 | } else { | |
220 | len = strlen(p); | |
221 | } | |
222 | ||
223 | if (strncmp(p, "id=", 3) == 0) { | |
224 | len -= 3; | |
225 | strncpy(clientname, p + 3, len); | |
226 | clientname[len] = '\0'; | |
227 | return clientname; | |
228 | } | |
229 | if (end == NULL) { | |
230 | break; | |
231 | } | |
232 | p = end + 1; | |
233 | } | |
234 | return NULL; | |
235 | } | |
236 | ||
fab5cf59 JD |
237 | static int qemu_rbd_set_conf(rados_t cluster, const char *conf) |
238 | { | |
239 | char *p, *buf; | |
240 | char name[RBD_MAX_CONF_NAME_SIZE]; | |
241 | char value[RBD_MAX_CONF_VAL_SIZE]; | |
242 | int ret = 0; | |
243 | ||
7267c094 | 244 | buf = g_strdup(conf); |
fab5cf59 JD |
245 | p = buf; |
246 | ||
247 | while (p) { | |
248 | ret = qemu_rbd_next_tok(name, sizeof(name), p, | |
249 | '=', "conf option name", &p); | |
250 | if (ret < 0) { | |
251 | break; | |
252 | } | |
16a06b24 | 253 | qemu_rbd_unescape(name); |
fab5cf59 JD |
254 | |
255 | if (!p) { | |
256 | error_report("conf option %s has no value", name); | |
257 | ret = -EINVAL; | |
258 | break; | |
259 | } | |
260 | ||
261 | ret = qemu_rbd_next_tok(value, sizeof(value), p, | |
262 | ':', "conf option value", &p); | |
263 | if (ret < 0) { | |
264 | break; | |
265 | } | |
16a06b24 | 266 | qemu_rbd_unescape(value); |
fab5cf59 | 267 | |
7c7e9df0 SW |
268 | if (strcmp(name, "conf") == 0) { |
269 | ret = rados_conf_read_file(cluster, value); | |
fab5cf59 | 270 | if (ret < 0) { |
7c7e9df0 | 271 | error_report("error reading conf file %s", value); |
fab5cf59 JD |
272 | break; |
273 | } | |
7c7e9df0 SW |
274 | } else if (strcmp(name, "id") == 0) { |
275 | /* ignore, this is parsed by qemu_rbd_parse_clientname() */ | |
fab5cf59 | 276 | } else { |
7c7e9df0 | 277 | ret = rados_conf_set(cluster, name, value); |
fab5cf59 | 278 | if (ret < 0) { |
7c7e9df0 SW |
279 | error_report("invalid conf option %s", name); |
280 | ret = -EINVAL; | |
fab5cf59 JD |
281 | break; |
282 | } | |
283 | } | |
284 | } | |
285 | ||
7267c094 | 286 | g_free(buf); |
fab5cf59 JD |
287 | return ret; |
288 | } | |
289 | ||
d5124c00 HR |
290 | static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options, |
291 | Error **errp) | |
f27aaf4b CB |
292 | { |
293 | int64_t bytes = 0; | |
294 | int64_t objsize; | |
ad32e9c0 JD |
295 | int obj_order = 0; |
296 | char pool[RBD_MAX_POOL_NAME_SIZE]; | |
297 | char name[RBD_MAX_IMAGE_NAME_SIZE]; | |
298 | char snap_buf[RBD_MAX_SNAP_NAME_SIZE]; | |
fab5cf59 | 299 | char conf[RBD_MAX_CONF_SIZE]; |
7c7e9df0 SW |
300 | char clientname_buf[RBD_MAX_CONF_SIZE]; |
301 | char *clientname; | |
ad32e9c0 JD |
302 | rados_t cluster; |
303 | rados_ioctx_t io_ctx; | |
f27aaf4b CB |
304 | int ret; |
305 | ||
ad32e9c0 JD |
306 | if (qemu_rbd_parsename(filename, pool, sizeof(pool), |
307 | snap_buf, sizeof(snap_buf), | |
fab5cf59 JD |
308 | name, sizeof(name), |
309 | conf, sizeof(conf)) < 0) { | |
f27aaf4b CB |
310 | return -EINVAL; |
311 | } | |
f27aaf4b | 312 | |
f27aaf4b CB |
313 | /* Read out options */ |
314 | while (options && options->name) { | |
315 | if (!strcmp(options->name, BLOCK_OPT_SIZE)) { | |
316 | bytes = options->value.n; | |
317 | } else if (!strcmp(options->name, BLOCK_OPT_CLUSTER_SIZE)) { | |
318 | if (options->value.n) { | |
319 | objsize = options->value.n; | |
320 | if ((objsize - 1) & objsize) { /* not a power of 2? */ | |
321 | error_report("obj size needs to be power of 2"); | |
322 | return -EINVAL; | |
323 | } | |
324 | if (objsize < 4096) { | |
325 | error_report("obj size too small"); | |
326 | return -EINVAL; | |
327 | } | |
ad32e9c0 | 328 | obj_order = ffs(objsize) - 1; |
f27aaf4b CB |
329 | } |
330 | } | |
331 | options++; | |
332 | } | |
333 | ||
7c7e9df0 SW |
334 | clientname = qemu_rbd_parse_clientname(conf, clientname_buf); |
335 | if (rados_create(&cluster, clientname) < 0) { | |
f27aaf4b CB |
336 | error_report("error initializing"); |
337 | return -EIO; | |
338 | } | |
339 | ||
fab5cf59 | 340 | if (strstr(conf, "conf=") == NULL) { |
f9fe18ec SW |
341 | /* try default location, but ignore failure */ |
342 | rados_conf_read_file(cluster, NULL); | |
fab5cf59 JD |
343 | } |
344 | ||
345 | if (conf[0] != '\0' && | |
346 | qemu_rbd_set_conf(cluster, conf) < 0) { | |
347 | error_report("error setting config options"); | |
ad32e9c0 | 348 | rados_shutdown(cluster); |
f27aaf4b CB |
349 | return -EIO; |
350 | } | |
351 | ||
ad32e9c0 JD |
352 | if (rados_connect(cluster) < 0) { |
353 | error_report("error connecting"); | |
354 | rados_shutdown(cluster); | |
f27aaf4b CB |
355 | return -EIO; |
356 | } | |
f27aaf4b | 357 | |
ad32e9c0 JD |
358 | if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) { |
359 | error_report("error opening pool %s", pool); | |
360 | rados_shutdown(cluster); | |
361 | return -EIO; | |
f27aaf4b CB |
362 | } |
363 | ||
ad32e9c0 JD |
364 | ret = rbd_create(io_ctx, name, bytes, &obj_order); |
365 | rados_ioctx_destroy(io_ctx); | |
366 | rados_shutdown(cluster); | |
f27aaf4b CB |
367 | |
368 | return ret; | |
369 | } | |
370 | ||
371 | /* | |
ad32e9c0 JD |
372 | * This aio completion is being called from qemu_rbd_aio_event_reader() |
373 | * and runs in qemu context. It schedules a bh, but just in case the aio | |
f27aaf4b CB |
374 | * was not cancelled before. |
375 | */ | |
ad32e9c0 | 376 | static void qemu_rbd_complete_aio(RADOSCB *rcb) |
f27aaf4b CB |
377 | { |
378 | RBDAIOCB *acb = rcb->acb; | |
379 | int64_t r; | |
380 | ||
f27aaf4b CB |
381 | r = rcb->ret; |
382 | ||
dc7588c1 | 383 | if (acb->cmd != RBD_AIO_READ) { |
f27aaf4b CB |
384 | if (r < 0) { |
385 | acb->ret = r; | |
386 | acb->error = 1; | |
387 | } else if (!acb->error) { | |
ad32e9c0 | 388 | acb->ret = rcb->size; |
f27aaf4b CB |
389 | } |
390 | } else { | |
ad32e9c0 JD |
391 | if (r < 0) { |
392 | memset(rcb->buf, 0, rcb->size); | |
f27aaf4b CB |
393 | acb->ret = r; |
394 | acb->error = 1; | |
ad32e9c0 JD |
395 | } else if (r < rcb->size) { |
396 | memset(rcb->buf + r, 0, rcb->size - r); | |
f27aaf4b | 397 | if (!acb->error) { |
ad32e9c0 | 398 | acb->ret = rcb->size; |
f27aaf4b CB |
399 | } |
400 | } else if (!acb->error) { | |
ad32e9c0 | 401 | acb->ret = r; |
f27aaf4b CB |
402 | } |
403 | } | |
404 | /* Note that acb->bh can be NULL in case where the aio was cancelled */ | |
ad32e9c0 JD |
405 | acb->bh = qemu_bh_new(rbd_aio_bh_cb, acb); |
406 | qemu_bh_schedule(acb->bh); | |
7267c094 | 407 | g_free(rcb); |
f27aaf4b CB |
408 | } |
409 | ||
410 | /* | |
411 | * aio fd read handler. It runs in the qemu context and calls the | |
412 | * completion handling of completed rados aio operations. | |
413 | */ | |
ad32e9c0 | 414 | static void qemu_rbd_aio_event_reader(void *opaque) |
f27aaf4b CB |
415 | { |
416 | BDRVRBDState *s = opaque; | |
417 | ||
418 | ssize_t ret; | |
419 | ||
420 | do { | |
421 | char *p = (char *)&s->event_rcb; | |
422 | ||
423 | /* now read the rcb pointer that was sent from a non qemu thread */ | |
dfe80b07 SW |
424 | ret = read(s->fds[RBD_FD_READ], p + s->event_reader_pos, |
425 | sizeof(s->event_rcb) - s->event_reader_pos); | |
426 | if (ret > 0) { | |
427 | s->event_reader_pos += ret; | |
428 | if (s->event_reader_pos == sizeof(s->event_rcb)) { | |
429 | s->event_reader_pos = 0; | |
430 | qemu_rbd_complete_aio(s->event_rcb); | |
f27aaf4b CB |
431 | } |
432 | } | |
433 | } while (ret < 0 && errno == EINTR); | |
434 | } | |
435 | ||
a9ccedc3 KW |
436 | /* TODO Convert to fine grained options */ |
437 | static QemuOptsList runtime_opts = { | |
438 | .name = "rbd", | |
439 | .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head), | |
440 | .desc = { | |
441 | { | |
442 | .name = "filename", | |
443 | .type = QEMU_OPT_STRING, | |
444 | .help = "Specification of the rbd image", | |
445 | }, | |
446 | { /* end of list */ } | |
447 | }, | |
448 | }; | |
449 | ||
015a1036 HR |
450 | static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags, |
451 | Error **errp) | |
f27aaf4b CB |
452 | { |
453 | BDRVRBDState *s = bs->opaque; | |
ad32e9c0 JD |
454 | char pool[RBD_MAX_POOL_NAME_SIZE]; |
455 | char snap_buf[RBD_MAX_SNAP_NAME_SIZE]; | |
fab5cf59 | 456 | char conf[RBD_MAX_CONF_SIZE]; |
7c7e9df0 SW |
457 | char clientname_buf[RBD_MAX_CONF_SIZE]; |
458 | char *clientname; | |
a9ccedc3 KW |
459 | QemuOpts *opts; |
460 | Error *local_err = NULL; | |
461 | const char *filename; | |
f27aaf4b CB |
462 | int r; |
463 | ||
a9ccedc3 KW |
464 | opts = qemu_opts_create_nofail(&runtime_opts); |
465 | qemu_opts_absorb_qdict(opts, options, &local_err); | |
466 | if (error_is_set(&local_err)) { | |
467 | qerror_report_err(local_err); | |
468 | error_free(local_err); | |
469 | qemu_opts_del(opts); | |
470 | return -EINVAL; | |
471 | } | |
472 | ||
473 | filename = qemu_opt_get(opts, "filename"); | |
a9ccedc3 | 474 | |
ad32e9c0 JD |
475 | if (qemu_rbd_parsename(filename, pool, sizeof(pool), |
476 | snap_buf, sizeof(snap_buf), | |
fab5cf59 JD |
477 | s->name, sizeof(s->name), |
478 | conf, sizeof(conf)) < 0) { | |
c3ca988d KW |
479 | r = -EINVAL; |
480 | goto failed_opts; | |
f27aaf4b | 481 | } |
f27aaf4b | 482 | |
7c7e9df0 SW |
483 | clientname = qemu_rbd_parse_clientname(conf, clientname_buf); |
484 | r = rados_create(&s->cluster, clientname); | |
ad32e9c0 | 485 | if (r < 0) { |
f27aaf4b | 486 | error_report("error initializing"); |
c3ca988d | 487 | goto failed_opts; |
f27aaf4b CB |
488 | } |
489 | ||
eb93d5d9 SW |
490 | s->snap = NULL; |
491 | if (snap_buf[0] != '\0') { | |
492 | s->snap = g_strdup(snap_buf); | |
493 | } | |
494 | ||
b11f38fc JD |
495 | /* |
496 | * Fallback to more conservative semantics if setting cache | |
497 | * options fails. Ignore errors from setting rbd_cache because the | |
498 | * only possible error is that the option does not exist, and | |
499 | * librbd defaults to no caching. If write through caching cannot | |
500 | * be set up, fall back to no caching. | |
501 | */ | |
502 | if (flags & BDRV_O_NOCACHE) { | |
503 | rados_conf_set(s->cluster, "rbd_cache", "false"); | |
504 | } else { | |
505 | rados_conf_set(s->cluster, "rbd_cache", "true"); | |
b11f38fc JD |
506 | } |
507 | ||
fab5cf59 | 508 | if (strstr(conf, "conf=") == NULL) { |
f9fe18ec SW |
509 | /* try default location, but ignore failure */ |
510 | rados_conf_read_file(s->cluster, NULL); | |
fab5cf59 JD |
511 | } |
512 | ||
513 | if (conf[0] != '\0') { | |
514 | r = qemu_rbd_set_conf(s->cluster, conf); | |
515 | if (r < 0) { | |
516 | error_report("error setting config options"); | |
eb93d5d9 | 517 | goto failed_shutdown; |
fab5cf59 | 518 | } |
f27aaf4b CB |
519 | } |
520 | ||
ad32e9c0 JD |
521 | r = rados_connect(s->cluster); |
522 | if (r < 0) { | |
523 | error_report("error connecting"); | |
eb93d5d9 | 524 | goto failed_shutdown; |
f27aaf4b CB |
525 | } |
526 | ||
ad32e9c0 JD |
527 | r = rados_ioctx_create(s->cluster, pool, &s->io_ctx); |
528 | if (r < 0) { | |
529 | error_report("error opening pool %s", pool); | |
eb93d5d9 | 530 | goto failed_shutdown; |
f27aaf4b CB |
531 | } |
532 | ||
ad32e9c0 | 533 | r = rbd_open(s->io_ctx, s->name, &s->image, s->snap); |
f27aaf4b | 534 | if (r < 0) { |
ad32e9c0 | 535 | error_report("error reading header from %s", s->name); |
eb93d5d9 | 536 | goto failed_open; |
f27aaf4b CB |
537 | } |
538 | ||
ad32e9c0 | 539 | bs->read_only = (s->snap != NULL); |
f27aaf4b CB |
540 | |
541 | s->event_reader_pos = 0; | |
542 | r = qemu_pipe(s->fds); | |
543 | if (r < 0) { | |
544 | error_report("error opening eventfd"); | |
545 | goto failed; | |
546 | } | |
547 | fcntl(s->fds[0], F_SETFL, O_NONBLOCK); | |
548 | fcntl(s->fds[1], F_SETFL, O_NONBLOCK); | |
ad32e9c0 | 549 | qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], qemu_rbd_aio_event_reader, |
f2e5dca4 | 550 | NULL, s); |
f27aaf4b | 551 | |
f27aaf4b | 552 | |
c3ca988d | 553 | qemu_opts_del(opts); |
f27aaf4b CB |
554 | return 0; |
555 | ||
556 | failed: | |
ad32e9c0 | 557 | rbd_close(s->image); |
eb93d5d9 | 558 | failed_open: |
ad32e9c0 | 559 | rados_ioctx_destroy(s->io_ctx); |
eb93d5d9 | 560 | failed_shutdown: |
ad32e9c0 | 561 | rados_shutdown(s->cluster); |
eb93d5d9 | 562 | g_free(s->snap); |
c3ca988d KW |
563 | failed_opts: |
564 | qemu_opts_del(opts); | |
f27aaf4b CB |
565 | return r; |
566 | } | |
567 | ||
ad32e9c0 | 568 | static void qemu_rbd_close(BlockDriverState *bs) |
f27aaf4b CB |
569 | { |
570 | BDRVRBDState *s = bs->opaque; | |
571 | ||
572 | close(s->fds[0]); | |
573 | close(s->fds[1]); | |
f2e5dca4 | 574 | qemu_aio_set_fd_handler(s->fds[RBD_FD_READ], NULL, NULL, NULL); |
f27aaf4b | 575 | |
ad32e9c0 JD |
576 | rbd_close(s->image); |
577 | rados_ioctx_destroy(s->io_ctx); | |
7267c094 | 578 | g_free(s->snap); |
ad32e9c0 | 579 | rados_shutdown(s->cluster); |
f27aaf4b CB |
580 | } |
581 | ||
582 | /* | |
583 | * Cancel aio. Since we don't reference acb in a non qemu threads, | |
584 | * it is safe to access it here. | |
585 | */ | |
ad32e9c0 | 586 | static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb) |
f27aaf4b CB |
587 | { |
588 | RBDAIOCB *acb = (RBDAIOCB *) blockacb; | |
589 | acb->cancelled = 1; | |
473c7f02 SP |
590 | |
591 | while (acb->status == -EINPROGRESS) { | |
592 | qemu_aio_wait(); | |
593 | } | |
594 | ||
595 | qemu_aio_release(acb); | |
f27aaf4b CB |
596 | } |
597 | ||
d7331bed | 598 | static const AIOCBInfo rbd_aiocb_info = { |
f27aaf4b | 599 | .aiocb_size = sizeof(RBDAIOCB), |
ad32e9c0 | 600 | .cancel = qemu_rbd_aio_cancel, |
f27aaf4b CB |
601 | }; |
602 | ||
ad32e9c0 | 603 | static int qemu_rbd_send_pipe(BDRVRBDState *s, RADOSCB *rcb) |
f27aaf4b | 604 | { |
ad32e9c0 | 605 | int ret = 0; |
f27aaf4b CB |
606 | while (1) { |
607 | fd_set wfd; | |
ad32e9c0 | 608 | int fd = s->fds[RBD_FD_WRITE]; |
f27aaf4b | 609 | |
ad32e9c0 JD |
610 | /* send the op pointer to the qemu thread that is responsible |
611 | for the aio/op completion. Must do it in a qemu thread context */ | |
f27aaf4b CB |
612 | ret = write(fd, (void *)&rcb, sizeof(rcb)); |
613 | if (ret >= 0) { | |
614 | break; | |
615 | } | |
616 | if (errno == EINTR) { | |
617 | continue; | |
ad32e9c0 | 618 | } |
f27aaf4b CB |
619 | if (errno != EAGAIN) { |
620 | break; | |
ad32e9c0 | 621 | } |
f27aaf4b CB |
622 | |
623 | FD_ZERO(&wfd); | |
624 | FD_SET(fd, &wfd); | |
625 | do { | |
626 | ret = select(fd + 1, NULL, &wfd, NULL, NULL); | |
627 | } while (ret < 0 && errno == EINTR); | |
628 | } | |
629 | ||
ad32e9c0 JD |
630 | return ret; |
631 | } | |
632 | ||
633 | /* | |
634 | * This is the callback function for rbd_aio_read and _write | |
635 | * | |
636 | * Note: this function is being called from a non qemu thread so | |
637 | * we need to be careful about what we do here. Generally we only | |
638 | * write to the block notification pipe, and do the rest of the | |
639 | * io completion handling from qemu_rbd_aio_event_reader() which | |
640 | * runs in a qemu context. | |
641 | */ | |
642 | static void rbd_finish_aiocb(rbd_completion_t c, RADOSCB *rcb) | |
643 | { | |
644 | int ret; | |
645 | rcb->ret = rbd_aio_get_return_value(c); | |
646 | rbd_aio_release(c); | |
647 | ret = qemu_rbd_send_pipe(rcb->s, rcb); | |
f27aaf4b | 648 | if (ret < 0) { |
ad32e9c0 | 649 | error_report("failed writing to acb->s->fds"); |
7267c094 | 650 | g_free(rcb); |
f27aaf4b CB |
651 | } |
652 | } | |
653 | ||
ad32e9c0 | 654 | /* Callback when all queued rbd_aio requests are complete */ |
f27aaf4b CB |
655 | |
656 | static void rbd_aio_bh_cb(void *opaque) | |
657 | { | |
658 | RBDAIOCB *acb = opaque; | |
659 | ||
787f3133 | 660 | if (acb->cmd == RBD_AIO_READ) { |
03396148 | 661 | qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size); |
f27aaf4b CB |
662 | } |
663 | qemu_vfree(acb->bounce); | |
664 | acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret)); | |
665 | qemu_bh_delete(acb->bh); | |
666 | acb->bh = NULL; | |
473c7f02 | 667 | acb->status = 0; |
f27aaf4b | 668 | |
473c7f02 SP |
669 | if (!acb->cancelled) { |
670 | qemu_aio_release(acb); | |
671 | } | |
f27aaf4b CB |
672 | } |
673 | ||
787f3133 JD |
674 | static int rbd_aio_discard_wrapper(rbd_image_t image, |
675 | uint64_t off, | |
676 | uint64_t len, | |
677 | rbd_completion_t comp) | |
678 | { | |
679 | #ifdef LIBRBD_SUPPORTS_DISCARD | |
680 | return rbd_aio_discard(image, off, len, comp); | |
681 | #else | |
682 | return -ENOTSUP; | |
683 | #endif | |
684 | } | |
685 | ||
dc7588c1 JD |
686 | static int rbd_aio_flush_wrapper(rbd_image_t image, |
687 | rbd_completion_t comp) | |
688 | { | |
689 | #ifdef LIBRBD_SUPPORTS_AIO_FLUSH | |
690 | return rbd_aio_flush(image, comp); | |
691 | #else | |
692 | return -ENOTSUP; | |
693 | #endif | |
694 | } | |
695 | ||
787f3133 JD |
696 | static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs, |
697 | int64_t sector_num, | |
698 | QEMUIOVector *qiov, | |
699 | int nb_sectors, | |
700 | BlockDriverCompletionFunc *cb, | |
701 | void *opaque, | |
702 | RBDAIOCmd cmd) | |
f27aaf4b CB |
703 | { |
704 | RBDAIOCB *acb; | |
705 | RADOSCB *rcb; | |
ad32e9c0 | 706 | rbd_completion_t c; |
f27aaf4b CB |
707 | int64_t off, size; |
708 | char *buf; | |
51a13528 | 709 | int r; |
f27aaf4b CB |
710 | |
711 | BDRVRBDState *s = bs->opaque; | |
712 | ||
d7331bed | 713 | acb = qemu_aio_get(&rbd_aiocb_info, bs, cb, opaque); |
787f3133 | 714 | acb->cmd = cmd; |
f27aaf4b | 715 | acb->qiov = qiov; |
dc7588c1 | 716 | if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) { |
787f3133 JD |
717 | acb->bounce = NULL; |
718 | } else { | |
719 | acb->bounce = qemu_blockalign(bs, qiov->size); | |
720 | } | |
f27aaf4b CB |
721 | acb->ret = 0; |
722 | acb->error = 0; | |
723 | acb->s = s; | |
724 | acb->cancelled = 0; | |
725 | acb->bh = NULL; | |
473c7f02 | 726 | acb->status = -EINPROGRESS; |
f27aaf4b | 727 | |
787f3133 | 728 | if (cmd == RBD_AIO_WRITE) { |
d5e6b161 | 729 | qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size); |
f27aaf4b CB |
730 | } |
731 | ||
732 | buf = acb->bounce; | |
733 | ||
734 | off = sector_num * BDRV_SECTOR_SIZE; | |
735 | size = nb_sectors * BDRV_SECTOR_SIZE; | |
f27aaf4b | 736 | |
7267c094 | 737 | rcb = g_malloc(sizeof(RADOSCB)); |
ad32e9c0 JD |
738 | rcb->done = 0; |
739 | rcb->acb = acb; | |
740 | rcb->buf = buf; | |
741 | rcb->s = acb->s; | |
742 | rcb->size = size; | |
51a13528 JD |
743 | r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c); |
744 | if (r < 0) { | |
745 | goto failed; | |
746 | } | |
f27aaf4b | 747 | |
787f3133 JD |
748 | switch (cmd) { |
749 | case RBD_AIO_WRITE: | |
51a13528 | 750 | r = rbd_aio_write(s->image, off, size, buf, c); |
787f3133 JD |
751 | break; |
752 | case RBD_AIO_READ: | |
51a13528 | 753 | r = rbd_aio_read(s->image, off, size, buf, c); |
787f3133 JD |
754 | break; |
755 | case RBD_AIO_DISCARD: | |
756 | r = rbd_aio_discard_wrapper(s->image, off, size, c); | |
757 | break; | |
dc7588c1 JD |
758 | case RBD_AIO_FLUSH: |
759 | r = rbd_aio_flush_wrapper(s->image, c); | |
760 | break; | |
787f3133 JD |
761 | default: |
762 | r = -EINVAL; | |
51a13528 JD |
763 | } |
764 | ||
765 | if (r < 0) { | |
766 | goto failed; | |
f27aaf4b CB |
767 | } |
768 | ||
769 | return &acb->common; | |
51a13528 JD |
770 | |
771 | failed: | |
7267c094 | 772 | g_free(rcb); |
51a13528 JD |
773 | qemu_aio_release(acb); |
774 | return NULL; | |
f27aaf4b CB |
775 | } |
776 | ||
ad32e9c0 JD |
777 | static BlockDriverAIOCB *qemu_rbd_aio_readv(BlockDriverState *bs, |
778 | int64_t sector_num, | |
779 | QEMUIOVector *qiov, | |
780 | int nb_sectors, | |
781 | BlockDriverCompletionFunc *cb, | |
782 | void *opaque) | |
f27aaf4b | 783 | { |
787f3133 JD |
784 | return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque, |
785 | RBD_AIO_READ); | |
f27aaf4b CB |
786 | } |
787 | ||
ad32e9c0 JD |
788 | static BlockDriverAIOCB *qemu_rbd_aio_writev(BlockDriverState *bs, |
789 | int64_t sector_num, | |
790 | QEMUIOVector *qiov, | |
791 | int nb_sectors, | |
792 | BlockDriverCompletionFunc *cb, | |
793 | void *opaque) | |
f27aaf4b | 794 | { |
787f3133 JD |
795 | return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque, |
796 | RBD_AIO_WRITE); | |
f27aaf4b CB |
797 | } |
798 | ||
dc7588c1 JD |
799 | #ifdef LIBRBD_SUPPORTS_AIO_FLUSH |
800 | static BlockDriverAIOCB *qemu_rbd_aio_flush(BlockDriverState *bs, | |
801 | BlockDriverCompletionFunc *cb, | |
802 | void *opaque) | |
803 | { | |
804 | return rbd_start_aio(bs, 0, NULL, 0, cb, opaque, RBD_AIO_FLUSH); | |
805 | } | |
806 | ||
807 | #else | |
808 | ||
8b94ff85 | 809 | static int qemu_rbd_co_flush(BlockDriverState *bs) |
7a3f5fe9 SW |
810 | { |
811 | #if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 1) | |
812 | /* rbd_flush added in 0.1.1 */ | |
813 | BDRVRBDState *s = bs->opaque; | |
814 | return rbd_flush(s->image); | |
815 | #else | |
816 | return 0; | |
817 | #endif | |
818 | } | |
dc7588c1 | 819 | #endif |
7a3f5fe9 | 820 | |
ad32e9c0 | 821 | static int qemu_rbd_getinfo(BlockDriverState *bs, BlockDriverInfo *bdi) |
f27aaf4b CB |
822 | { |
823 | BDRVRBDState *s = bs->opaque; | |
ad32e9c0 JD |
824 | rbd_image_info_t info; |
825 | int r; | |
826 | ||
827 | r = rbd_stat(s->image, &info, sizeof(info)); | |
828 | if (r < 0) { | |
829 | return r; | |
830 | } | |
831 | ||
832 | bdi->cluster_size = info.obj_size; | |
f27aaf4b CB |
833 | return 0; |
834 | } | |
835 | ||
ad32e9c0 | 836 | static int64_t qemu_rbd_getlength(BlockDriverState *bs) |
f27aaf4b CB |
837 | { |
838 | BDRVRBDState *s = bs->opaque; | |
ad32e9c0 JD |
839 | rbd_image_info_t info; |
840 | int r; | |
f27aaf4b | 841 | |
ad32e9c0 JD |
842 | r = rbd_stat(s->image, &info, sizeof(info)); |
843 | if (r < 0) { | |
844 | return r; | |
845 | } | |
846 | ||
847 | return info.size; | |
f27aaf4b CB |
848 | } |
849 | ||
30cdc48c JD |
850 | static int qemu_rbd_truncate(BlockDriverState *bs, int64_t offset) |
851 | { | |
852 | BDRVRBDState *s = bs->opaque; | |
853 | int r; | |
854 | ||
855 | r = rbd_resize(s->image, offset); | |
856 | if (r < 0) { | |
857 | return r; | |
858 | } | |
859 | ||
860 | return 0; | |
861 | } | |
862 | ||
ad32e9c0 JD |
863 | static int qemu_rbd_snap_create(BlockDriverState *bs, |
864 | QEMUSnapshotInfo *sn_info) | |
f27aaf4b CB |
865 | { |
866 | BDRVRBDState *s = bs->opaque; | |
f27aaf4b | 867 | int r; |
f27aaf4b CB |
868 | |
869 | if (sn_info->name[0] == '\0') { | |
870 | return -EINVAL; /* we need a name for rbd snapshots */ | |
871 | } | |
872 | ||
873 | /* | |
874 | * rbd snapshots are using the name as the user controlled unique identifier | |
875 | * we can't use the rbd snapid for that purpose, as it can't be set | |
876 | */ | |
877 | if (sn_info->id_str[0] != '\0' && | |
878 | strcmp(sn_info->id_str, sn_info->name) != 0) { | |
879 | return -EINVAL; | |
880 | } | |
881 | ||
882 | if (strlen(sn_info->name) >= sizeof(sn_info->id_str)) { | |
883 | return -ERANGE; | |
884 | } | |
885 | ||
ad32e9c0 | 886 | r = rbd_snap_create(s->image, sn_info->name); |
f27aaf4b | 887 | if (r < 0) { |
ad32e9c0 | 888 | error_report("failed to create snap: %s", strerror(-r)); |
f27aaf4b CB |
889 | return r; |
890 | } | |
891 | ||
f27aaf4b CB |
892 | return 0; |
893 | } | |
894 | ||
bd603247 | 895 | static int qemu_rbd_snap_remove(BlockDriverState *bs, |
a89d89d3 WX |
896 | const char *snapshot_id, |
897 | const char *snapshot_name, | |
898 | Error **errp) | |
bd603247 GF |
899 | { |
900 | BDRVRBDState *s = bs->opaque; | |
901 | int r; | |
902 | ||
a89d89d3 WX |
903 | if (!snapshot_name) { |
904 | error_setg(errp, "rbd need a valid snapshot name"); | |
905 | return -EINVAL; | |
906 | } | |
907 | ||
908 | /* If snapshot_id is specified, it must be equal to name, see | |
909 | qemu_rbd_snap_list() */ | |
910 | if (snapshot_id && strcmp(snapshot_id, snapshot_name)) { | |
911 | error_setg(errp, | |
912 | "rbd do not support snapshot id, it should be NULL or " | |
913 | "equal to snapshot name"); | |
914 | return -EINVAL; | |
915 | } | |
916 | ||
bd603247 | 917 | r = rbd_snap_remove(s->image, snapshot_name); |
a89d89d3 WX |
918 | if (r < 0) { |
919 | error_setg_errno(errp, -r, "Failed to remove the snapshot"); | |
920 | } | |
bd603247 GF |
921 | return r; |
922 | } | |
923 | ||
924 | static int qemu_rbd_snap_rollback(BlockDriverState *bs, | |
925 | const char *snapshot_name) | |
926 | { | |
927 | BDRVRBDState *s = bs->opaque; | |
928 | int r; | |
929 | ||
930 | r = rbd_snap_rollback(s->image, snapshot_name); | |
931 | return r; | |
932 | } | |
933 | ||
ad32e9c0 JD |
934 | static int qemu_rbd_snap_list(BlockDriverState *bs, |
935 | QEMUSnapshotInfo **psn_tab) | |
f27aaf4b CB |
936 | { |
937 | BDRVRBDState *s = bs->opaque; | |
f27aaf4b | 938 | QEMUSnapshotInfo *sn_info, *sn_tab = NULL; |
ad32e9c0 JD |
939 | int i, snap_count; |
940 | rbd_snap_info_t *snaps; | |
941 | int max_snaps = RBD_MAX_SNAPS; | |
f27aaf4b | 942 | |
ad32e9c0 | 943 | do { |
7267c094 | 944 | snaps = g_malloc(sizeof(*snaps) * max_snaps); |
ad32e9c0 JD |
945 | snap_count = rbd_snap_list(s->image, snaps, &max_snaps); |
946 | if (snap_count < 0) { | |
7267c094 | 947 | g_free(snaps); |
f27aaf4b | 948 | } |
ad32e9c0 | 949 | } while (snap_count == -ERANGE); |
f27aaf4b | 950 | |
ad32e9c0 | 951 | if (snap_count <= 0) { |
b9c53290 | 952 | goto done; |
f27aaf4b CB |
953 | } |
954 | ||
7267c094 | 955 | sn_tab = g_malloc0(snap_count * sizeof(QEMUSnapshotInfo)); |
f27aaf4b | 956 | |
ad32e9c0 JD |
957 | for (i = 0; i < snap_count; i++) { |
958 | const char *snap_name = snaps[i].name; | |
f27aaf4b CB |
959 | |
960 | sn_info = sn_tab + i; | |
961 | pstrcpy(sn_info->id_str, sizeof(sn_info->id_str), snap_name); | |
962 | pstrcpy(sn_info->name, sizeof(sn_info->name), snap_name); | |
f27aaf4b | 963 | |
ad32e9c0 | 964 | sn_info->vm_state_size = snaps[i].size; |
f27aaf4b CB |
965 | sn_info->date_sec = 0; |
966 | sn_info->date_nsec = 0; | |
967 | sn_info->vm_clock_nsec = 0; | |
968 | } | |
ad32e9c0 JD |
969 | rbd_snap_list_end(snaps); |
970 | ||
b9c53290 | 971 | done: |
f27aaf4b | 972 | *psn_tab = sn_tab; |
f27aaf4b | 973 | return snap_count; |
f27aaf4b CB |
974 | } |
975 | ||
787f3133 JD |
976 | #ifdef LIBRBD_SUPPORTS_DISCARD |
977 | static BlockDriverAIOCB* qemu_rbd_aio_discard(BlockDriverState *bs, | |
978 | int64_t sector_num, | |
979 | int nb_sectors, | |
980 | BlockDriverCompletionFunc *cb, | |
981 | void *opaque) | |
982 | { | |
983 | return rbd_start_aio(bs, sector_num, NULL, nb_sectors, cb, opaque, | |
984 | RBD_AIO_DISCARD); | |
985 | } | |
986 | #endif | |
987 | ||
ad32e9c0 | 988 | static QEMUOptionParameter qemu_rbd_create_options[] = { |
f27aaf4b CB |
989 | { |
990 | .name = BLOCK_OPT_SIZE, | |
991 | .type = OPT_SIZE, | |
992 | .help = "Virtual disk size" | |
993 | }, | |
994 | { | |
995 | .name = BLOCK_OPT_CLUSTER_SIZE, | |
996 | .type = OPT_SIZE, | |
997 | .help = "RBD object size" | |
998 | }, | |
999 | {NULL} | |
1000 | }; | |
1001 | ||
1002 | static BlockDriver bdrv_rbd = { | |
1003 | .format_name = "rbd", | |
1004 | .instance_size = sizeof(BDRVRBDState), | |
030be321 | 1005 | .bdrv_needs_filename = true, |
ad32e9c0 JD |
1006 | .bdrv_file_open = qemu_rbd_open, |
1007 | .bdrv_close = qemu_rbd_close, | |
1008 | .bdrv_create = qemu_rbd_create, | |
3ac21627 | 1009 | .bdrv_has_zero_init = bdrv_has_zero_init_1, |
ad32e9c0 JD |
1010 | .bdrv_get_info = qemu_rbd_getinfo, |
1011 | .create_options = qemu_rbd_create_options, | |
1012 | .bdrv_getlength = qemu_rbd_getlength, | |
30cdc48c | 1013 | .bdrv_truncate = qemu_rbd_truncate, |
f27aaf4b CB |
1014 | .protocol_name = "rbd", |
1015 | ||
c68b89ac KW |
1016 | .bdrv_aio_readv = qemu_rbd_aio_readv, |
1017 | .bdrv_aio_writev = qemu_rbd_aio_writev, | |
dc7588c1 JD |
1018 | |
1019 | #ifdef LIBRBD_SUPPORTS_AIO_FLUSH | |
1020 | .bdrv_aio_flush = qemu_rbd_aio_flush, | |
1021 | #else | |
c68b89ac | 1022 | .bdrv_co_flush_to_disk = qemu_rbd_co_flush, |
dc7588c1 | 1023 | #endif |
f27aaf4b | 1024 | |
787f3133 JD |
1025 | #ifdef LIBRBD_SUPPORTS_DISCARD |
1026 | .bdrv_aio_discard = qemu_rbd_aio_discard, | |
1027 | #endif | |
1028 | ||
c68b89ac | 1029 | .bdrv_snapshot_create = qemu_rbd_snap_create, |
bd603247 | 1030 | .bdrv_snapshot_delete = qemu_rbd_snap_remove, |
c68b89ac | 1031 | .bdrv_snapshot_list = qemu_rbd_snap_list, |
bd603247 | 1032 | .bdrv_snapshot_goto = qemu_rbd_snap_rollback, |
f27aaf4b CB |
1033 | }; |
1034 | ||
1035 | static void bdrv_rbd_init(void) | |
1036 | { | |
1037 | bdrv_register(&bdrv_rbd); | |
1038 | } | |
1039 | ||
1040 | block_init(bdrv_rbd_init); |