2 * GlusterFS backend for QEMU
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
10 #include "qemu/osdep.h"
11 #include <glusterfs/api/glfs.h>
12 #include "block/block_int.h"
15 typedef struct GlusterAIOCB {
20 AioContext *aio_context;
23 typedef struct BDRVGlusterState {
28 typedef struct GlusterConf {
36 static void qemu_gluster_gconf_free(GlusterConf *gconf)
39 g_free(gconf->server);
40 g_free(gconf->volname);
42 g_free(gconf->transport);
47 static int parse_volume_options(GlusterConf *gconf, char *path)
56 p = q = path + strspn(path, "/");
61 gconf->volname = g_strndup(q, p - q);
68 gconf->image = g_strdup(p);
73 * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
75 * 'gluster' is the protocol.
77 * 'transport' specifies the transport type used to connect to gluster
78 * management daemon (glusterd). Valid transport types are
79 * tcp, unix and rdma. If a transport type isn't specified, then tcp
82 * 'server' specifies the server where the volume file specification for
83 * the given volume resides. This can be either hostname, ipv4 address
84 * or ipv6 address. ipv6 address needs to be within square brackets [ ].
85 * If transport type is 'unix', then 'server' field should not be specified.
86 * The 'socket' field needs to be populated with the path to unix domain
89 * 'port' is the port number on which glusterd is listening. This is optional
90 * and if not specified, QEMU will send 0 which will make gluster to use the
91 * default port. If the transport type is unix, then 'port' should not be
94 * 'volname' is the name of the gluster volume which contains the VM image.
96 * 'image' is the path to the actual VM image that resides on gluster volume.
100 * file=gluster://1.2.3.4/testvol/a.img
101 * file=gluster+tcp://1.2.3.4/testvol/a.img
102 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
103 * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
104 * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
105 * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
106 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
107 * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
109 static int qemu_gluster_parseuri(GlusterConf *gconf, const char *filename)
112 QueryParams *qp = NULL;
113 bool is_unix = false;
116 uri = uri_parse(filename);
122 if (!uri->scheme || !strcmp(uri->scheme, "gluster")) {
123 gconf->transport = g_strdup("tcp");
124 } else if (!strcmp(uri->scheme, "gluster+tcp")) {
125 gconf->transport = g_strdup("tcp");
126 } else if (!strcmp(uri->scheme, "gluster+unix")) {
127 gconf->transport = g_strdup("unix");
129 } else if (!strcmp(uri->scheme, "gluster+rdma")) {
130 gconf->transport = g_strdup("rdma");
136 ret = parse_volume_options(gconf, uri->path);
141 qp = query_params_parse(uri->query);
142 if (qp->n > 1 || (is_unix && !qp->n) || (!is_unix && qp->n)) {
148 if (uri->server || uri->port) {
152 if (strcmp(qp->p[0].name, "socket")) {
156 gconf->server = g_strdup(qp->p[0].value);
158 gconf->server = g_strdup(uri->server ? uri->server : "localhost");
159 gconf->port = uri->port;
164 query_params_free(qp);
170 static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename,
173 struct glfs *glfs = NULL;
177 ret = qemu_gluster_parseuri(gconf, filename);
179 error_setg(errp, "Usage: file=gluster[+transport]://[server[:port]]/"
180 "volname/image[?socket=...]");
185 glfs = glfs_new(gconf->volname);
190 ret = glfs_set_volfile_server(glfs, gconf->transport, gconf->server,
197 * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
198 * GlusterFS makes GF_LOG_* macros available to libgfapi users.
200 ret = glfs_set_logging(glfs, "-", 4);
205 ret = glfs_init(glfs);
207 error_setg_errno(errp, errno,
208 "Gluster connection failed for server=%s port=%d "
209 "volume=%s image=%s transport=%s", gconf->server,
210 gconf->port, gconf->volname, gconf->image,
213 /* glfs_init sometimes doesn't set errno although docs suggest that */
230 static void qemu_gluster_complete_aio(void *opaque)
232 GlusterAIOCB *acb = (GlusterAIOCB *)opaque;
234 qemu_bh_delete(acb->bh);
236 qemu_coroutine_enter(acb->coroutine, NULL);
240 * AIO callback routine called from GlusterFS thread.
242 static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)
244 GlusterAIOCB *acb = (GlusterAIOCB *)arg;
246 if (!ret || ret == acb->size) {
247 acb->ret = 0; /* Success */
248 } else if (ret < 0) {
249 acb->ret = ret; /* Read/Write failed */
251 acb->ret = -EIO; /* Partial read/write - fail it */
254 acb->bh = aio_bh_new(acb->aio_context, qemu_gluster_complete_aio, acb);
255 qemu_bh_schedule(acb->bh);
258 /* TODO Convert to fine grained options */
259 static QemuOptsList runtime_opts = {
261 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
265 .type = QEMU_OPT_STRING,
266 .help = "URL to the gluster image",
268 { /* end of list */ }
272 static void qemu_gluster_parse_flags(int bdrv_flags, int *open_flags)
274 assert(open_flags != NULL);
276 *open_flags |= O_BINARY;
278 if (bdrv_flags & BDRV_O_RDWR) {
279 *open_flags |= O_RDWR;
281 *open_flags |= O_RDONLY;
284 if ((bdrv_flags & BDRV_O_NOCACHE)) {
285 *open_flags |= O_DIRECT;
289 static int qemu_gluster_open(BlockDriverState *bs, QDict *options,
290 int bdrv_flags, Error **errp)
292 BDRVGlusterState *s = bs->opaque;
295 GlusterConf *gconf = g_new0(GlusterConf, 1);
297 Error *local_err = NULL;
298 const char *filename;
300 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
301 qemu_opts_absorb_qdict(opts, options, &local_err);
303 error_propagate(errp, local_err);
308 filename = qemu_opt_get(opts, "filename");
310 s->glfs = qemu_gluster_init(gconf, filename, errp);
316 qemu_gluster_parse_flags(bdrv_flags, &open_flags);
318 s->fd = glfs_open(s->glfs, gconf->image, open_flags);
325 qemu_gluster_gconf_free(gconf);
338 typedef struct BDRVGlusterReopenState {
341 } BDRVGlusterReopenState;
344 static int qemu_gluster_reopen_prepare(BDRVReopenState *state,
345 BlockReopenQueue *queue, Error **errp)
348 BDRVGlusterReopenState *reop_s;
349 GlusterConf *gconf = NULL;
352 assert(state != NULL);
353 assert(state->bs != NULL);
355 state->opaque = g_new0(BDRVGlusterReopenState, 1);
356 reop_s = state->opaque;
358 qemu_gluster_parse_flags(state->flags, &open_flags);
360 gconf = g_new0(GlusterConf, 1);
362 reop_s->glfs = qemu_gluster_init(gconf, state->bs->filename, errp);
363 if (reop_s->glfs == NULL) {
368 reop_s->fd = glfs_open(reop_s->glfs, gconf->image, open_flags);
369 if (reop_s->fd == NULL) {
370 /* reops->glfs will be cleaned up in _abort */
376 /* state->opaque will be freed in either the _abort or _commit */
377 qemu_gluster_gconf_free(gconf);
381 static void qemu_gluster_reopen_commit(BDRVReopenState *state)
383 BDRVGlusterReopenState *reop_s = state->opaque;
384 BDRVGlusterState *s = state->bs->opaque;
395 /* use the newly opened image / connection */
397 s->glfs = reop_s->glfs;
399 g_free(state->opaque);
400 state->opaque = NULL;
406 static void qemu_gluster_reopen_abort(BDRVReopenState *state)
408 BDRVGlusterReopenState *reop_s = state->opaque;
410 if (reop_s == NULL) {
415 glfs_close(reop_s->fd);
419 glfs_fini(reop_s->glfs);
422 g_free(state->opaque);
423 state->opaque = NULL;
428 #ifdef CONFIG_GLUSTERFS_ZEROFILL
429 static coroutine_fn int qemu_gluster_co_write_zeroes(BlockDriverState *bs,
430 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
434 BDRVGlusterState *s = bs->opaque;
435 off_t size = nb_sectors * BDRV_SECTOR_SIZE;
436 off_t offset = sector_num * BDRV_SECTOR_SIZE;
440 acb.coroutine = qemu_coroutine_self();
441 acb.aio_context = bdrv_get_aio_context(bs);
443 ret = glfs_zerofill_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
448 qemu_coroutine_yield();
452 static inline bool gluster_supports_zerofill(void)
457 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
460 return glfs_zerofill(fd, offset, size);
464 static inline bool gluster_supports_zerofill(void)
469 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
476 static int qemu_gluster_create(const char *filename,
477 QemuOpts *opts, Error **errp)
483 int64_t total_size = 0;
485 GlusterConf *gconf = g_new0(GlusterConf, 1);
487 glfs = qemu_gluster_init(gconf, filename, errp);
493 total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
496 tmp = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
497 if (!tmp || !strcmp(tmp, "off")) {
499 } else if (!strcmp(tmp, "full") &&
500 gluster_supports_zerofill()) {
503 error_setg(errp, "Invalid preallocation mode: '%s'"
504 " or GlusterFS doesn't support zerofill API",
510 fd = glfs_creat(glfs, gconf->image,
511 O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
515 if (!glfs_ftruncate(fd, total_size)) {
516 if (prealloc && qemu_gluster_zerofill(fd, 0, total_size)) {
523 if (glfs_close(fd) != 0) {
529 qemu_gluster_gconf_free(gconf);
536 static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,
537 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int write)
541 BDRVGlusterState *s = bs->opaque;
542 size_t size = nb_sectors * BDRV_SECTOR_SIZE;
543 off_t offset = sector_num * BDRV_SECTOR_SIZE;
547 acb.coroutine = qemu_coroutine_self();
548 acb.aio_context = bdrv_get_aio_context(bs);
551 ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
552 gluster_finish_aiocb, &acb);
554 ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
555 gluster_finish_aiocb, &acb);
562 qemu_coroutine_yield();
566 static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset)
569 BDRVGlusterState *s = bs->opaque;
571 ret = glfs_ftruncate(s->fd, offset);
579 static coroutine_fn int qemu_gluster_co_readv(BlockDriverState *bs,
580 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
582 return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 0);
585 static coroutine_fn int qemu_gluster_co_writev(BlockDriverState *bs,
586 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
588 return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 1);
591 static coroutine_fn int qemu_gluster_co_flush_to_disk(BlockDriverState *bs)
595 BDRVGlusterState *s = bs->opaque;
599 acb.coroutine = qemu_coroutine_self();
600 acb.aio_context = bdrv_get_aio_context(bs);
602 ret = glfs_fsync_async(s->fd, gluster_finish_aiocb, &acb);
607 qemu_coroutine_yield();
611 #ifdef CONFIG_GLUSTERFS_DISCARD
612 static coroutine_fn int qemu_gluster_co_discard(BlockDriverState *bs,
613 int64_t sector_num, int nb_sectors)
617 BDRVGlusterState *s = bs->opaque;
618 size_t size = nb_sectors * BDRV_SECTOR_SIZE;
619 off_t offset = sector_num * BDRV_SECTOR_SIZE;
623 acb.coroutine = qemu_coroutine_self();
624 acb.aio_context = bdrv_get_aio_context(bs);
626 ret = glfs_discard_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
631 qemu_coroutine_yield();
636 static int64_t qemu_gluster_getlength(BlockDriverState *bs)
638 BDRVGlusterState *s = bs->opaque;
641 ret = glfs_lseek(s->fd, 0, SEEK_END);
649 static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs)
651 BDRVGlusterState *s = bs->opaque;
655 ret = glfs_fstat(s->fd, &st);
659 return st.st_blocks * 512;
663 static void qemu_gluster_close(BlockDriverState *bs)
665 BDRVGlusterState *s = bs->opaque;
674 static int qemu_gluster_has_zero_init(BlockDriverState *bs)
676 /* GlusterFS volume could be backed by a block device */
680 static QemuOptsList qemu_gluster_create_opts = {
681 .name = "qemu-gluster-create-opts",
682 .head = QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts.head),
685 .name = BLOCK_OPT_SIZE,
686 .type = QEMU_OPT_SIZE,
687 .help = "Virtual disk size"
690 .name = BLOCK_OPT_PREALLOC,
691 .type = QEMU_OPT_STRING,
692 .help = "Preallocation mode (allowed values: off, full)"
694 { /* end of list */ }
698 static BlockDriver bdrv_gluster = {
699 .format_name = "gluster",
700 .protocol_name = "gluster",
701 .instance_size = sizeof(BDRVGlusterState),
702 .bdrv_needs_filename = true,
703 .bdrv_file_open = qemu_gluster_open,
704 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
705 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
706 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
707 .bdrv_close = qemu_gluster_close,
708 .bdrv_create = qemu_gluster_create,
709 .bdrv_getlength = qemu_gluster_getlength,
710 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
711 .bdrv_truncate = qemu_gluster_truncate,
712 .bdrv_co_readv = qemu_gluster_co_readv,
713 .bdrv_co_writev = qemu_gluster_co_writev,
714 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
715 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
716 #ifdef CONFIG_GLUSTERFS_DISCARD
717 .bdrv_co_discard = qemu_gluster_co_discard,
719 #ifdef CONFIG_GLUSTERFS_ZEROFILL
720 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
722 .create_opts = &qemu_gluster_create_opts,
725 static BlockDriver bdrv_gluster_tcp = {
726 .format_name = "gluster",
727 .protocol_name = "gluster+tcp",
728 .instance_size = sizeof(BDRVGlusterState),
729 .bdrv_needs_filename = true,
730 .bdrv_file_open = qemu_gluster_open,
731 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
732 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
733 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
734 .bdrv_close = qemu_gluster_close,
735 .bdrv_create = qemu_gluster_create,
736 .bdrv_getlength = qemu_gluster_getlength,
737 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
738 .bdrv_truncate = qemu_gluster_truncate,
739 .bdrv_co_readv = qemu_gluster_co_readv,
740 .bdrv_co_writev = qemu_gluster_co_writev,
741 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
742 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
743 #ifdef CONFIG_GLUSTERFS_DISCARD
744 .bdrv_co_discard = qemu_gluster_co_discard,
746 #ifdef CONFIG_GLUSTERFS_ZEROFILL
747 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
749 .create_opts = &qemu_gluster_create_opts,
752 static BlockDriver bdrv_gluster_unix = {
753 .format_name = "gluster",
754 .protocol_name = "gluster+unix",
755 .instance_size = sizeof(BDRVGlusterState),
756 .bdrv_needs_filename = true,
757 .bdrv_file_open = qemu_gluster_open,
758 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
759 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
760 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
761 .bdrv_close = qemu_gluster_close,
762 .bdrv_create = qemu_gluster_create,
763 .bdrv_getlength = qemu_gluster_getlength,
764 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
765 .bdrv_truncate = qemu_gluster_truncate,
766 .bdrv_co_readv = qemu_gluster_co_readv,
767 .bdrv_co_writev = qemu_gluster_co_writev,
768 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
769 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
770 #ifdef CONFIG_GLUSTERFS_DISCARD
771 .bdrv_co_discard = qemu_gluster_co_discard,
773 #ifdef CONFIG_GLUSTERFS_ZEROFILL
774 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
776 .create_opts = &qemu_gluster_create_opts,
779 static BlockDriver bdrv_gluster_rdma = {
780 .format_name = "gluster",
781 .protocol_name = "gluster+rdma",
782 .instance_size = sizeof(BDRVGlusterState),
783 .bdrv_needs_filename = true,
784 .bdrv_file_open = qemu_gluster_open,
785 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
786 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
787 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
788 .bdrv_close = qemu_gluster_close,
789 .bdrv_create = qemu_gluster_create,
790 .bdrv_getlength = qemu_gluster_getlength,
791 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
792 .bdrv_truncate = qemu_gluster_truncate,
793 .bdrv_co_readv = qemu_gluster_co_readv,
794 .bdrv_co_writev = qemu_gluster_co_writev,
795 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
796 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
797 #ifdef CONFIG_GLUSTERFS_DISCARD
798 .bdrv_co_discard = qemu_gluster_co_discard,
800 #ifdef CONFIG_GLUSTERFS_ZEROFILL
801 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
803 .create_opts = &qemu_gluster_create_opts,
806 static void bdrv_gluster_init(void)
808 bdrv_register(&bdrv_gluster_rdma);
809 bdrv_register(&bdrv_gluster_unix);
810 bdrv_register(&bdrv_gluster_tcp);
811 bdrv_register(&bdrv_gluster);
814 block_init(bdrv_gluster_init);