2 * GlusterFS backend for QEMU
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
10 #include <glusterfs/api/glfs.h>
11 #include "block/block_int.h"
14 typedef struct GlusterAIOCB {
19 AioContext *aio_context;
22 typedef struct BDRVGlusterState {
27 typedef struct GlusterConf {
35 static void qemu_gluster_gconf_free(GlusterConf *gconf)
38 g_free(gconf->server);
39 g_free(gconf->volname);
41 g_free(gconf->transport);
46 static int parse_volume_options(GlusterConf *gconf, char *path)
55 p = q = path + strspn(path, "/");
60 gconf->volname = g_strndup(q, p - q);
67 gconf->image = g_strdup(p);
72 * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
74 * 'gluster' is the protocol.
76 * 'transport' specifies the transport type used to connect to gluster
77 * management daemon (glusterd). Valid transport types are
78 * tcp, unix and rdma. If a transport type isn't specified, then tcp
81 * 'server' specifies the server where the volume file specification for
82 * the given volume resides. This can be either hostname, ipv4 address
83 * or ipv6 address. ipv6 address needs to be within square brackets [ ].
84 * If transport type is 'unix', then 'server' field should not be specified.
85 * The 'socket' field needs to be populated with the path to unix domain
88 * 'port' is the port number on which glusterd is listening. This is optional
89 * and if not specified, QEMU will send 0 which will make gluster to use the
90 * default port. If the transport type is unix, then 'port' should not be
93 * 'volname' is the name of the gluster volume which contains the VM image.
95 * 'image' is the path to the actual VM image that resides on gluster volume.
99 * file=gluster://1.2.3.4/testvol/a.img
100 * file=gluster+tcp://1.2.3.4/testvol/a.img
101 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
102 * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
103 * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
104 * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
105 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
106 * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
108 static int qemu_gluster_parseuri(GlusterConf *gconf, const char *filename)
111 QueryParams *qp = NULL;
112 bool is_unix = false;
115 uri = uri_parse(filename);
121 if (!uri->scheme || !strcmp(uri->scheme, "gluster")) {
122 gconf->transport = g_strdup("tcp");
123 } else if (!strcmp(uri->scheme, "gluster+tcp")) {
124 gconf->transport = g_strdup("tcp");
125 } else if (!strcmp(uri->scheme, "gluster+unix")) {
126 gconf->transport = g_strdup("unix");
128 } else if (!strcmp(uri->scheme, "gluster+rdma")) {
129 gconf->transport = g_strdup("rdma");
135 ret = parse_volume_options(gconf, uri->path);
140 qp = query_params_parse(uri->query);
141 if (qp->n > 1 || (is_unix && !qp->n) || (!is_unix && qp->n)) {
147 if (uri->server || uri->port) {
151 if (strcmp(qp->p[0].name, "socket")) {
155 gconf->server = g_strdup(qp->p[0].value);
157 gconf->server = g_strdup(uri->server ? uri->server : "localhost");
158 gconf->port = uri->port;
163 query_params_free(qp);
169 static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename,
172 struct glfs *glfs = NULL;
176 ret = qemu_gluster_parseuri(gconf, filename);
178 error_setg(errp, "Usage: file=gluster[+transport]://[server[:port]]/"
179 "volname/image[?socket=...]");
184 glfs = glfs_new(gconf->volname);
189 ret = glfs_set_volfile_server(glfs, gconf->transport, gconf->server,
196 * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
197 * GlusterFS makes GF_LOG_* macros available to libgfapi users.
199 ret = glfs_set_logging(glfs, "-", 4);
204 ret = glfs_init(glfs);
206 error_setg_errno(errp, errno,
207 "Gluster connection failed for server=%s port=%d "
208 "volume=%s image=%s transport=%s", gconf->server,
209 gconf->port, gconf->volname, gconf->image,
212 /* glfs_init sometimes doesn't set errno although docs suggest that */
229 static void qemu_gluster_complete_aio(void *opaque)
231 GlusterAIOCB *acb = (GlusterAIOCB *)opaque;
233 qemu_bh_delete(acb->bh);
235 qemu_coroutine_enter(acb->coroutine, NULL);
239 * AIO callback routine called from GlusterFS thread.
241 static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)
243 GlusterAIOCB *acb = (GlusterAIOCB *)arg;
245 if (!ret || ret == acb->size) {
246 acb->ret = 0; /* Success */
247 } else if (ret < 0) {
248 acb->ret = ret; /* Read/Write failed */
250 acb->ret = -EIO; /* Partial read/write - fail it */
253 acb->bh = aio_bh_new(acb->aio_context, qemu_gluster_complete_aio, acb);
254 qemu_bh_schedule(acb->bh);
257 /* TODO Convert to fine grained options */
258 static QemuOptsList runtime_opts = {
260 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
264 .type = QEMU_OPT_STRING,
265 .help = "URL to the gluster image",
267 { /* end of list */ }
271 static void qemu_gluster_parse_flags(int bdrv_flags, int *open_flags)
273 assert(open_flags != NULL);
275 *open_flags |= O_BINARY;
277 if (bdrv_flags & BDRV_O_RDWR) {
278 *open_flags |= O_RDWR;
280 *open_flags |= O_RDONLY;
283 if ((bdrv_flags & BDRV_O_NOCACHE)) {
284 *open_flags |= O_DIRECT;
288 static int qemu_gluster_open(BlockDriverState *bs, QDict *options,
289 int bdrv_flags, Error **errp)
291 BDRVGlusterState *s = bs->opaque;
294 GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
296 Error *local_err = NULL;
297 const char *filename;
299 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
300 qemu_opts_absorb_qdict(opts, options, &local_err);
302 error_propagate(errp, local_err);
307 filename = qemu_opt_get(opts, "filename");
309 s->glfs = qemu_gluster_init(gconf, filename, errp);
315 qemu_gluster_parse_flags(bdrv_flags, &open_flags);
317 s->fd = glfs_open(s->glfs, gconf->image, open_flags);
324 qemu_gluster_gconf_free(gconf);
337 typedef struct BDRVGlusterReopenState {
340 } BDRVGlusterReopenState;
343 static int qemu_gluster_reopen_prepare(BDRVReopenState *state,
344 BlockReopenQueue *queue, Error **errp)
347 BDRVGlusterReopenState *reop_s;
348 GlusterConf *gconf = NULL;
351 assert(state != NULL);
352 assert(state->bs != NULL);
354 state->opaque = g_malloc0(sizeof(BDRVGlusterReopenState));
355 reop_s = state->opaque;
357 qemu_gluster_parse_flags(state->flags, &open_flags);
359 gconf = g_malloc0(sizeof(GlusterConf));
361 reop_s->glfs = qemu_gluster_init(gconf, state->bs->filename, errp);
362 if (reop_s->glfs == NULL) {
367 reop_s->fd = glfs_open(reop_s->glfs, gconf->image, open_flags);
368 if (reop_s->fd == NULL) {
369 /* reops->glfs will be cleaned up in _abort */
375 /* state->opaque will be freed in either the _abort or _commit */
376 qemu_gluster_gconf_free(gconf);
380 static void qemu_gluster_reopen_commit(BDRVReopenState *state)
382 BDRVGlusterReopenState *reop_s = state->opaque;
383 BDRVGlusterState *s = state->bs->opaque;
394 /* use the newly opened image / connection */
396 s->glfs = reop_s->glfs;
398 g_free(state->opaque);
399 state->opaque = NULL;
405 static void qemu_gluster_reopen_abort(BDRVReopenState *state)
407 BDRVGlusterReopenState *reop_s = state->opaque;
409 if (reop_s == NULL) {
414 glfs_close(reop_s->fd);
418 glfs_fini(reop_s->glfs);
421 g_free(state->opaque);
422 state->opaque = NULL;
427 #ifdef CONFIG_GLUSTERFS_ZEROFILL
428 static coroutine_fn int qemu_gluster_co_write_zeroes(BlockDriverState *bs,
429 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
432 GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
433 BDRVGlusterState *s = bs->opaque;
434 off_t size = nb_sectors * BDRV_SECTOR_SIZE;
435 off_t offset = sector_num * BDRV_SECTOR_SIZE;
439 acb->coroutine = qemu_coroutine_self();
440 acb->aio_context = bdrv_get_aio_context(bs);
442 ret = glfs_zerofill_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
448 qemu_coroutine_yield();
452 g_slice_free(GlusterAIOCB, acb);
456 static inline bool gluster_supports_zerofill(void)
461 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
464 return glfs_zerofill(fd, offset, size);
468 static inline bool gluster_supports_zerofill(void)
473 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
480 static int qemu_gluster_create(const char *filename,
481 QemuOpts *opts, Error **errp)
487 int64_t total_size = 0;
489 GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
491 glfs = qemu_gluster_init(gconf, filename, errp);
498 qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0) / BDRV_SECTOR_SIZE;
500 tmp = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
501 if (!tmp || !strcmp(tmp, "off")) {
503 } else if (!strcmp(tmp, "full") &&
504 gluster_supports_zerofill()) {
507 error_setg(errp, "Invalid preallocation mode: '%s'"
508 " or GlusterFS doesn't support zerofill API",
514 fd = glfs_creat(glfs, gconf->image,
515 O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
519 if (!glfs_ftruncate(fd, total_size * BDRV_SECTOR_SIZE)) {
520 if (prealloc && qemu_gluster_zerofill(fd, 0,
521 total_size * BDRV_SECTOR_SIZE)) {
528 if (glfs_close(fd) != 0) {
534 qemu_gluster_gconf_free(gconf);
541 static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,
542 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int write)
545 GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
546 BDRVGlusterState *s = bs->opaque;
547 size_t size = nb_sectors * BDRV_SECTOR_SIZE;
548 off_t offset = sector_num * BDRV_SECTOR_SIZE;
552 acb->coroutine = qemu_coroutine_self();
553 acb->aio_context = bdrv_get_aio_context(bs);
556 ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
557 &gluster_finish_aiocb, acb);
559 ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
560 &gluster_finish_aiocb, acb);
568 qemu_coroutine_yield();
572 g_slice_free(GlusterAIOCB, acb);
576 static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset)
579 BDRVGlusterState *s = bs->opaque;
581 ret = glfs_ftruncate(s->fd, offset);
589 static coroutine_fn int qemu_gluster_co_readv(BlockDriverState *bs,
590 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
592 return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 0);
595 static coroutine_fn int qemu_gluster_co_writev(BlockDriverState *bs,
596 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
598 return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 1);
601 static coroutine_fn int qemu_gluster_co_flush_to_disk(BlockDriverState *bs)
604 GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
605 BDRVGlusterState *s = bs->opaque;
609 acb->coroutine = qemu_coroutine_self();
610 acb->aio_context = bdrv_get_aio_context(bs);
612 ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb);
618 qemu_coroutine_yield();
622 g_slice_free(GlusterAIOCB, acb);
626 #ifdef CONFIG_GLUSTERFS_DISCARD
627 static coroutine_fn int qemu_gluster_co_discard(BlockDriverState *bs,
628 int64_t sector_num, int nb_sectors)
631 GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
632 BDRVGlusterState *s = bs->opaque;
633 size_t size = nb_sectors * BDRV_SECTOR_SIZE;
634 off_t offset = sector_num * BDRV_SECTOR_SIZE;
638 acb->coroutine = qemu_coroutine_self();
639 acb->aio_context = bdrv_get_aio_context(bs);
641 ret = glfs_discard_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
647 qemu_coroutine_yield();
651 g_slice_free(GlusterAIOCB, acb);
656 static int64_t qemu_gluster_getlength(BlockDriverState *bs)
658 BDRVGlusterState *s = bs->opaque;
661 ret = glfs_lseek(s->fd, 0, SEEK_END);
669 static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs)
671 BDRVGlusterState *s = bs->opaque;
675 ret = glfs_fstat(s->fd, &st);
679 return st.st_blocks * 512;
683 static void qemu_gluster_close(BlockDriverState *bs)
685 BDRVGlusterState *s = bs->opaque;
694 static int qemu_gluster_has_zero_init(BlockDriverState *bs)
696 /* GlusterFS volume could be backed by a block device */
700 static QemuOptsList qemu_gluster_create_opts = {
701 .name = "qemu-gluster-create-opts",
702 .head = QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts.head),
705 .name = BLOCK_OPT_SIZE,
706 .type = QEMU_OPT_SIZE,
707 .help = "Virtual disk size"
710 .name = BLOCK_OPT_PREALLOC,
711 .type = QEMU_OPT_STRING,
712 .help = "Preallocation mode (allowed values: off, full)"
714 { /* end of list */ }
718 static BlockDriver bdrv_gluster = {
719 .format_name = "gluster",
720 .protocol_name = "gluster",
721 .instance_size = sizeof(BDRVGlusterState),
722 .bdrv_needs_filename = true,
723 .bdrv_file_open = qemu_gluster_open,
724 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
725 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
726 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
727 .bdrv_close = qemu_gluster_close,
728 .bdrv_create = qemu_gluster_create,
729 .bdrv_getlength = qemu_gluster_getlength,
730 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
731 .bdrv_truncate = qemu_gluster_truncate,
732 .bdrv_co_readv = qemu_gluster_co_readv,
733 .bdrv_co_writev = qemu_gluster_co_writev,
734 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
735 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
736 #ifdef CONFIG_GLUSTERFS_DISCARD
737 .bdrv_co_discard = qemu_gluster_co_discard,
739 #ifdef CONFIG_GLUSTERFS_ZEROFILL
740 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
742 .create_opts = &qemu_gluster_create_opts,
745 static BlockDriver bdrv_gluster_tcp = {
746 .format_name = "gluster",
747 .protocol_name = "gluster+tcp",
748 .instance_size = sizeof(BDRVGlusterState),
749 .bdrv_needs_filename = true,
750 .bdrv_file_open = qemu_gluster_open,
751 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
752 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
753 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
754 .bdrv_close = qemu_gluster_close,
755 .bdrv_create = qemu_gluster_create,
756 .bdrv_getlength = qemu_gluster_getlength,
757 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
758 .bdrv_truncate = qemu_gluster_truncate,
759 .bdrv_co_readv = qemu_gluster_co_readv,
760 .bdrv_co_writev = qemu_gluster_co_writev,
761 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
762 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
763 #ifdef CONFIG_GLUSTERFS_DISCARD
764 .bdrv_co_discard = qemu_gluster_co_discard,
766 #ifdef CONFIG_GLUSTERFS_ZEROFILL
767 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
769 .create_opts = &qemu_gluster_create_opts,
772 static BlockDriver bdrv_gluster_unix = {
773 .format_name = "gluster",
774 .protocol_name = "gluster+unix",
775 .instance_size = sizeof(BDRVGlusterState),
776 .bdrv_needs_filename = true,
777 .bdrv_file_open = qemu_gluster_open,
778 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
779 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
780 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
781 .bdrv_close = qemu_gluster_close,
782 .bdrv_create = qemu_gluster_create,
783 .bdrv_getlength = qemu_gluster_getlength,
784 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
785 .bdrv_truncate = qemu_gluster_truncate,
786 .bdrv_co_readv = qemu_gluster_co_readv,
787 .bdrv_co_writev = qemu_gluster_co_writev,
788 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
789 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
790 #ifdef CONFIG_GLUSTERFS_DISCARD
791 .bdrv_co_discard = qemu_gluster_co_discard,
793 #ifdef CONFIG_GLUSTERFS_ZEROFILL
794 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
796 .create_opts = &qemu_gluster_create_opts,
799 static BlockDriver bdrv_gluster_rdma = {
800 .format_name = "gluster",
801 .protocol_name = "gluster+rdma",
802 .instance_size = sizeof(BDRVGlusterState),
803 .bdrv_needs_filename = true,
804 .bdrv_file_open = qemu_gluster_open,
805 .bdrv_reopen_prepare = qemu_gluster_reopen_prepare,
806 .bdrv_reopen_commit = qemu_gluster_reopen_commit,
807 .bdrv_reopen_abort = qemu_gluster_reopen_abort,
808 .bdrv_close = qemu_gluster_close,
809 .bdrv_create = qemu_gluster_create,
810 .bdrv_getlength = qemu_gluster_getlength,
811 .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
812 .bdrv_truncate = qemu_gluster_truncate,
813 .bdrv_co_readv = qemu_gluster_co_readv,
814 .bdrv_co_writev = qemu_gluster_co_writev,
815 .bdrv_co_flush_to_disk = qemu_gluster_co_flush_to_disk,
816 .bdrv_has_zero_init = qemu_gluster_has_zero_init,
817 #ifdef CONFIG_GLUSTERFS_DISCARD
818 .bdrv_co_discard = qemu_gluster_co_discard,
820 #ifdef CONFIG_GLUSTERFS_ZEROFILL
821 .bdrv_co_write_zeroes = qemu_gluster_co_write_zeroes,
823 .create_opts = &qemu_gluster_create_opts,
826 static void bdrv_gluster_init(void)
828 bdrv_register(&bdrv_gluster_rdma);
829 bdrv_register(&bdrv_gluster_unix);
830 bdrv_register(&bdrv_gluster_tcp);
831 bdrv_register(&bdrv_gluster);
834 block_init(bdrv_gluster_init);