[qemu.git] / block / qcow2-threads.c

/*
 * Threaded data processing for Qcow2: compression, encryption
 *
 * Copyright (c) 2004-2006 Fabrice Bellard
 * Copyright (c) 2018 Virtuozzo International GmbH. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include "qemu/osdep.h"

#define ZLIB_CONST
#include <zlib.h>

#ifdef CONFIG_ZSTD
#include <zstd.h>
#include <zstd_errors.h>
#endif

#include "qcow2.h"
#include "block/thread-pool.h"
#include "crypto.h"

static int coroutine_fn
qcow2_co_process(BlockDriverState *bs, ThreadPoolFunc *func, void *arg)
{
    int ret;
    BDRVQcow2State *s = bs->opaque;
    ThreadPool *pool = aio_get_thread_pool(bdrv_get_aio_context(bs));

    qemu_co_mutex_lock(&s->lock);
    while (s->nb_threads >= QCOW2_MAX_THREADS) {
        qemu_co_queue_wait(&s->thread_task_queue, &s->lock);
    }
    s->nb_threads++;
    qemu_co_mutex_unlock(&s->lock);

    ret = thread_pool_submit_co(pool, func, arg);

    qemu_co_mutex_lock(&s->lock);
    s->nb_threads--;
    qemu_co_queue_next(&s->thread_task_queue);
    qemu_co_mutex_unlock(&s->lock);

    return ret;
}


/*
 * Compression
 */

typedef ssize_t (*Qcow2CompressFunc)(void *dest, size_t dest_size,
                                     const void *src, size_t src_size);
typedef struct Qcow2CompressData {
    void *dest;
    size_t dest_size;
    const void *src;
    size_t src_size;
    ssize_t ret;

    Qcow2CompressFunc func;
} Qcow2CompressData;

/*
 * qcow2_zlib_compress()
 *
 * Compress @src_size bytes of data using zlib compression method
 *
 * @dest - destination buffer, @dest_size bytes
 * @src - source buffer, @src_size bytes
 *
 * Returns: compressed size on success
 *          -ENOMEM destination buffer is not enough to store compressed data
 *          -EIO    on any other error
 */
static ssize_t qcow2_zlib_compress(void *dest, size_t dest_size,
                                   const void *src, size_t src_size)
{
    ssize_t ret;
    z_stream strm;

    /* best compression, small window, no zlib header */
    memset(&strm, 0, sizeof(strm));
    ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
                       -12, 9, Z_DEFAULT_STRATEGY);
    if (ret != Z_OK) {
        return -EIO;
    }

    /*
     * strm.next_in is not const in old zlib versions, such as those used on
     * OpenBSD/NetBSD, so cast the const away
     */
    strm.avail_in = src_size;
    strm.next_in = (void *) src;
    strm.avail_out = dest_size;
    strm.next_out = dest;

    ret = deflate(&strm, Z_FINISH);
    if (ret == Z_STREAM_END) {
        ret = dest_size - strm.avail_out;
    } else {
        ret = (ret == Z_OK ? -ENOMEM : -EIO);
    }

    deflateEnd(&strm);

    return ret;
}

/*
 * qcow2_zlib_decompress()
 *
 * Decompress some data (not more than @src_size bytes) to produce exactly
 * @dest_size bytes using zlib compression method
 *
 * @dest - destination buffer, @dest_size bytes
 * @src - source buffer, @src_size bytes
 *
 * Returns: 0 on success
 *          -EIO on fail
 */
static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
                                     const void *src, size_t src_size)
{
    int ret;
    z_stream strm;

    memset(&strm, 0, sizeof(strm));
    strm.avail_in = src_size;
    strm.next_in = (void *) src;
    strm.avail_out = dest_size;
    strm.next_out = dest;

    ret = inflateInit2(&strm, -12);
    if (ret != Z_OK) {
        return -EIO;
    }

    ret = inflate(&strm, Z_FINISH);
    if ((ret == Z_STREAM_END || ret == Z_BUF_ERROR) && strm.avail_out == 0) {
        /*
         * We approve Z_BUF_ERROR because we need @dest buffer to be filled, but
         * @src buffer may be processed partly (because in qcow2 we know size of
         * compressed data with precision of one sector)
         */
        ret = 0;
    } else {
        ret = -EIO;
    }

    inflateEnd(&strm);

    return ret;
}

#ifdef CONFIG_ZSTD

/*
 * qcow2_zstd_compress()
 *
 * Compress @src_size bytes of data using zstd compression method
 *
 * @dest - destination buffer, @dest_size bytes
 * @src - source buffer, @src_size bytes
 *
 * Returns: compressed size on success
 *          -ENOMEM destination buffer is not enough to store compressed data
 *          -EIO    on any other error
 */
static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
                                   const void *src, size_t src_size)
{
    ssize_t ret;
    size_t zstd_ret;
    ZSTD_outBuffer output = {
        .dst = dest,
        .size = dest_size,
        .pos = 0
    };
    ZSTD_inBuffer input = {
        .src = src,
        .size = src_size,
        .pos = 0
    };
    ZSTD_CCtx *cctx = ZSTD_createCCtx();

    if (!cctx) {
        return -EIO;
    }
    /*
     * Use the zstd streamed interface for symmetry with decompression,
     * where streaming is essential since we don't record the exact
     * compressed size.
     *
     * ZSTD_compressStream2() tries to compress everything it could
     * with a single call. Although, ZSTD docs says that:
     * "You must continue calling ZSTD_compressStream2() with ZSTD_e_end
     * until it returns 0, at which point you are free to start a new frame",
     * in out tests we saw the only case when it returned with >0 -
     * when the output buffer was too small. In that case,
     * ZSTD_compressStream2() expects a bigger buffer on the next call.
     * We can't provide a bigger buffer because we are limited with dest_size
     * which we pass to the ZSTD_compressStream2() at once.
     * So, we don't need any loops and just abort the compression when we
     * don't get 0 result on the first call.
     */
    zstd_ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);

    if (zstd_ret) {
        if (zstd_ret > output.size - output.pos) {
            ret = -ENOMEM;
        } else {
            ret = -EIO;
        }
        goto out;
    }

    /* make sure that zstd didn't overflow the dest buffer */
    assert(output.pos <= dest_size);
    ret = output.pos;
out:
    ZSTD_freeCCtx(cctx);
    return ret;
}

/*
 * qcow2_zstd_decompress()
 *
 * Decompress some data (not more than @src_size bytes) to produce exactly
 * @dest_size bytes using zstd compression method
 *
 * @dest - destination buffer, @dest_size bytes
 * @src - source buffer, @src_size bytes
 *
 * Returns: 0 on success
 *          -EIO on any error
 */
static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
                                     const void *src, size_t src_size)
{
    size_t zstd_ret = 0;
    ssize_t ret = 0;
    ZSTD_outBuffer output = {
        .dst = dest,
        .size = dest_size,
        .pos = 0
    };
    ZSTD_inBuffer input = {
        .src = src,
        .size = src_size,
        .pos = 0
    };
    ZSTD_DCtx *dctx = ZSTD_createDCtx();

    if (!dctx) {
        return -EIO;
    }

    /*
     * The compressed stream from the input buffer may consist of more
     * than one zstd frame. So we iterate until we get a fully
     * uncompressed cluster.
     * From zstd docs related to ZSTD_decompressStream:
     * "return : 0 when a frame is completely decoded and fully flushed"
     * We suppose that this means: each time ZSTD_decompressStream reads
     * only ONE full frame and returns 0 if and only if that frame
     * is completely decoded and flushed. Only after returning 0,
     * ZSTD_decompressStream reads another ONE full frame.
     */
    while (output.pos < output.size) {
        size_t last_in_pos = input.pos;
        size_t last_out_pos = output.pos;
        zstd_ret = ZSTD_decompressStream(dctx, &output, &input);

        if (ZSTD_isError(zstd_ret)) {
            ret = -EIO;
            break;
        }

        /*
         * The ZSTD manual is vague about what to do if it reads
         * the buffer partially, and we don't want to get stuck
         * in an infinite loop where ZSTD_decompressStream
         * returns > 0 waiting for another input chunk. So, we add
         * a check which ensures that the loop makes some progress
         * on each step.
         */
        if (last_in_pos >= input.pos &&
            last_out_pos >= output.pos) {
            ret = -EIO;
            break;
        }
    }
    /*
     * Make sure that we have the frame fully flushed here
     * if not, we somehow managed to get uncompressed cluster
     * greater then the cluster size, possibly because of its
     * damage.
     */
    if (zstd_ret > 0) {
        ret = -EIO;
    }

    ZSTD_freeDCtx(dctx);
    assert(ret == 0 || ret == -EIO);
    return ret;
}
#endif

static int qcow2_compress_pool_func(void *opaque)
{
    Qcow2CompressData *data = opaque;

    data->ret = data->func(data->dest, data->dest_size,
                           data->src, data->src_size);

    return 0;
}

static ssize_t coroutine_fn
qcow2_co_do_compress(BlockDriverState *bs, void *dest, size_t dest_size,
                     const void *src, size_t src_size, Qcow2CompressFunc func)
{
    Qcow2CompressData arg = {
        .dest = dest,
        .dest_size = dest_size,
        .src = src,
        .src_size = src_size,
        .func = func,
    };

    qcow2_co_process(bs, qcow2_compress_pool_func, &arg);

    return arg.ret;
}

/*
 * qcow2_co_compress()
 *
 * Compress @src_size bytes of data using the compression
 * method defined by the image compression type
 *
 * @dest - destination buffer, @dest_size bytes
 * @src - source buffer, @src_size bytes
 *
 * Returns: compressed size on success
 *          a negative error code on failure
 */
ssize_t coroutine_fn
qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
                  const void *src, size_t src_size)
{
    BDRVQcow2State *s = bs->opaque;
    Qcow2CompressFunc fn;

    switch (s->compression_type) {
    case QCOW2_COMPRESSION_TYPE_ZLIB:
        fn = qcow2_zlib_compress;
        break;

#ifdef CONFIG_ZSTD
    case QCOW2_COMPRESSION_TYPE_ZSTD:
        fn = qcow2_zstd_compress;
        break;
#endif
    default:
        abort();
    }

    return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
}

/*
 * qcow2_co_decompress()
 *
 * Decompress some data (not more than @src_size bytes) to produce exactly
 * @dest_size bytes using the compression method defined by the image
 * compression type
 *
 * @dest - destination buffer, @dest_size bytes
 * @src - source buffer, @src_size bytes
 *
 * Returns: 0 on success
 *          a negative error code on failure
 */
ssize_t coroutine_fn
qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
                    const void *src, size_t src_size)
{
    BDRVQcow2State *s = bs->opaque;
    Qcow2CompressFunc fn;

    switch (s->compression_type) {
    case QCOW2_COMPRESSION_TYPE_ZLIB:
        fn = qcow2_zlib_decompress;
        break;

#ifdef CONFIG_ZSTD
    case QCOW2_COMPRESSION_TYPE_ZSTD:
        fn = qcow2_zstd_decompress;
        break;
#endif
    default:
        abort();
    }

    return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
}


/*
 * Cryptography
 */

/*
 * Qcow2EncDecFunc: common prototype of qcrypto_block_encrypt() and
 * qcrypto_block_decrypt() functions.
 */
typedef int (*Qcow2EncDecFunc)(QCryptoBlock *block, uint64_t offset,
                               uint8_t *buf, size_t len, Error **errp);

typedef struct Qcow2EncDecData {
    QCryptoBlock *block;
    uint64_t offset;
    uint8_t *buf;
    size_t len;

    Qcow2EncDecFunc func;
} Qcow2EncDecData;

static int qcow2_encdec_pool_func(void *opaque)
{
    Qcow2EncDecData *data = opaque;

    return data->func(data->block, data->offset, data->buf, data->len, NULL);
}

static int coroutine_fn
qcow2_co_encdec(BlockDriverState *bs, uint64_t host_offset,
                uint64_t guest_offset, void *buf, size_t len,
                Qcow2EncDecFunc func)
{
    BDRVQcow2State *s = bs->opaque;
    Qcow2EncDecData arg = {
        .block = s->crypto,
        .offset = s->crypt_physical_offset ? host_offset : guest_offset,
        .buf = buf,
        .len = len,
        .func = func,
    };
    uint64_t sector_size;

    assert(s->crypto);

    sector_size = qcrypto_block_get_sector_size(s->crypto);
    assert(QEMU_IS_ALIGNED(guest_offset, sector_size));
    assert(QEMU_IS_ALIGNED(host_offset, sector_size));
    assert(QEMU_IS_ALIGNED(len, sector_size));

    return len == 0 ? 0 : qcow2_co_process(bs, qcow2_encdec_pool_func, &arg);
}

/*
 * qcow2_co_encrypt()
 *
 * Encrypts one or more contiguous aligned sectors
 *
 * @host_offset - underlying storage offset of the first sector of the
 * data to be encrypted
 *
 * @guest_offset - guest (virtual) offset of the first sector of the
 * data to be encrypted
 *
 * @buf - buffer with the data to encrypt, that after encryption
 *        will be written to the underlying storage device at
 *        @host_offset
 *
 * @len - length of the buffer (must be a multiple of the encryption
 *        sector size)
 *
 * Depending on the encryption method, @host_offset and/or @guest_offset
 * may be used for generating the initialization vector for
 * encryption.
 *
 * Note that while the whole range must be aligned on sectors, it
 * does not have to be aligned on clusters and can also cross cluster
 * boundaries
 */
int coroutine_fn
qcow2_co_encrypt(BlockDriverState *bs, uint64_t host_offset,
                 uint64_t guest_offset, void *buf, size_t len)
{
    return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,
                           qcrypto_block_encrypt);
}

/*
 * qcow2_co_decrypt()
 *
 * Decrypts one or more contiguous aligned sectors
 * Similar to qcow2_co_encrypt
 */
int coroutine_fn
qcow2_co_decrypt(BlockDriverState *bs, uint64_t host_offset,
                 uint64_t guest_offset, void *buf, size_t len)
{
    return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,
                           qcrypto_block_decrypt);
}
Commit	Line	Data
56e2f1d8 VSO	1	/*
	2	* Threaded data processing for Qcow2: compression, encryption
	3	*
	4	* Copyright (c) 2004-2006 Fabrice Bellard
	5	* Copyright (c) 2018 Virtuozzo International GmbH. All rights reserved.
	6	*
	7	* Permission is hereby granted, free of charge, to any person obtaining a copy
	8	* of this software and associated documentation files (the "Software"), to deal
	9	* in the Software without restriction, including without limitation the rights
	10	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	11	* copies of the Software, and to permit persons to whom the Software is
	12	* furnished to do so, subject to the following conditions:
	13	*
	14	* The above copyright notice and this permission notice shall be included in
	15	* all copies or substantial portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	18	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	19	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	20	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	21	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	22	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	23	* THE SOFTWARE.
	24	*/
	25
	26	#include "qemu/osdep.h"
	27
	28	#define ZLIB_CONST
	29	#include <zlib.h>
	30
d298ac10 DP	31	#ifdef CONFIG_ZSTD
	32	#include <zstd.h>
	33	#include <zstd_errors.h>
	34	#endif
	35
56e2f1d8 VSO	36	#include "qcow2.h"
56e2f1d8 VSO	37	#include "block/thread-pool.h"
8ac0f15f	38	#include "crypto.h"
6f13a316 VSO	39
	40	static int coroutine_fn
	41	qcow2_co_process(BlockDriverState bs, ThreadPoolFunc func, void *arg)
	42	{
	43	int ret;
	44	BDRVQcow2State *s = bs->opaque;
	45	ThreadPool *pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
	46
	47	qemu_co_mutex_lock(&s->lock);
	48	while (s->nb_threads >= QCOW2_MAX_THREADS) {
	49	qemu_co_queue_wait(&s->thread_task_queue, &s->lock);
	50	}
	51	s->nb_threads++;
	52	qemu_co_mutex_unlock(&s->lock);
	53
	54	ret = thread_pool_submit_co(pool, func, arg);
	55
	56	qemu_co_mutex_lock(&s->lock);
	57	s->nb_threads--;
	58	qemu_co_queue_next(&s->thread_task_queue);
	59	qemu_co_mutex_unlock(&s->lock);
	60
	61	return ret;
	62	}
	63
	64
	65	/*
	66	* Compression
	67	*/
56e2f1d8 VSO	68
	69	typedef ssize_t (Qcow2CompressFunc)(void dest, size_t dest_size,
	70	const void *src, size_t src_size);
	71	typedef struct Qcow2CompressData {
	72	void *dest;
	73	size_t dest_size;
	74	const void *src;
	75	size_t src_size;
	76	ssize_t ret;
	77
	78	Qcow2CompressFunc func;
	79	} Qcow2CompressData;
	80
	81	/*
25dd077d DP	82	* qcow2_zlib_compress()
	83	*
	84	* Compress @src_size bytes of data using zlib compression method
56e2f1d8 VSO	85	*
	86	* @dest - destination buffer, @dest_size bytes
	87	* @src - source buffer, @src_size bytes
	88	*
	89	* Returns: compressed size on success
	90	* -ENOMEM destination buffer is not enough to store compressed data
	91	* -EIO on any other error
	92	*/
25dd077d DP	93	static ssize_t qcow2_zlib_compress(void *dest, size_t dest_size,
25dd077d DP	94	const void *src, size_t src_size)
56e2f1d8 VSO	95	{
	96	ssize_t ret;
	97	z_stream strm;
	98
	99	/* best compression, small window, no zlib header */
	100	memset(&strm, 0, sizeof(strm));
	101	ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
	102	-12, 9, Z_DEFAULT_STRATEGY);
	103	if (ret != Z_OK) {
	104	return -EIO;
	105	}
	106
	107	/*
	108	* strm.next_in is not const in old zlib versions, such as those used on
	109	* OpenBSD/NetBSD, so cast the const away
	110	*/
	111	strm.avail_in = src_size;
	112	strm.next_in = (void *) src;
	113	strm.avail_out = dest_size;
	114	strm.next_out = dest;
	115
	116	ret = deflate(&strm, Z_FINISH);
	117	if (ret == Z_STREAM_END) {
	118	ret = dest_size - strm.avail_out;
	119	} else {
	120	ret = (ret == Z_OK ? -ENOMEM : -EIO);
	121	}
	122
	123	deflateEnd(&strm);
	124
	125	return ret;
	126	}
	127
	128	/*
25dd077d	129	* qcow2_zlib_decompress()
56e2f1d8 VSO	130	*
56e2f1d8 VSO	131	* Decompress some data (not more than @src_size bytes) to produce exactly
25dd077d	132	* @dest_size bytes using zlib compression method
56e2f1d8 VSO	133	*
	134	* @dest - destination buffer, @dest_size bytes
	135	* @src - source buffer, @src_size bytes
	136	*
	137	* Returns: 0 on success
e7266570	138	* -EIO on fail
56e2f1d8	139	*/
25dd077d DP	140	static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
25dd077d DP	141	const void *src, size_t src_size)
56e2f1d8	142	{
e7266570	143	int ret;
56e2f1d8 VSO	144	z_stream strm;
	145
	146	memset(&strm, 0, sizeof(strm));
	147	strm.avail_in = src_size;
	148	strm.next_in = (void *) src;
	149	strm.avail_out = dest_size;
	150	strm.next_out = dest;
	151
	152	ret = inflateInit2(&strm, -12);
	153	if (ret != Z_OK) {
e7266570	154	return -EIO;
56e2f1d8 VSO	155	}
	156
	157	ret = inflate(&strm, Z_FINISH);
e7266570	158	if ((ret == Z_STREAM_END \|\| ret == Z_BUF_ERROR) && strm.avail_out == 0) {
56e2f1d8 VSO	159	/*
	160	* We approve Z_BUF_ERROR because we need @dest buffer to be filled, but
	161	* @src buffer may be processed partly (because in qcow2 we know size of
	162	* compressed data with precision of one sector)
	163	*/
e7266570 VSO	164	ret = 0;
	165	} else {
	166	ret = -EIO;
56e2f1d8 VSO	167	}
	168
	169	inflateEnd(&strm);
	170
	171	return ret;
	172	}
	173
d298ac10 DP	174	#ifdef CONFIG_ZSTD
	175
	176	/*
	177	* qcow2_zstd_compress()
	178	*
	179	* Compress @src_size bytes of data using zstd compression method
	180	*
	181	* @dest - destination buffer, @dest_size bytes
	182	* @src - source buffer, @src_size bytes
	183	*
	184	* Returns: compressed size on success
	185	* -ENOMEM destination buffer is not enough to store compressed data
	186	* -EIO on any other error
	187	*/
	188	static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
	189	const void *src, size_t src_size)
	190	{
	191	ssize_t ret;
	192	size_t zstd_ret;
	193	ZSTD_outBuffer output = {
	194	.dst = dest,
	195	.size = dest_size,
	196	.pos = 0
	197	};
	198	ZSTD_inBuffer input = {
	199	.src = src,
	200	.size = src_size,
	201	.pos = 0
	202	};
	203	ZSTD_CCtx *cctx = ZSTD_createCCtx();
	204
	205	if (!cctx) {
	206	return -EIO;
	207	}
	208	/*
	209	* Use the zstd streamed interface for symmetry with decompression,
	210	* where streaming is essential since we don't record the exact
	211	* compressed size.
	212	*
	213	* ZSTD_compressStream2() tries to compress everything it could
	214	* with a single call. Although, ZSTD docs says that:
	215	* "You must continue calling ZSTD_compressStream2() with ZSTD_e_end
	216	* until it returns 0, at which point you are free to start a new frame",
	217	* in out tests we saw the only case when it returned with >0 -
	218	* when the output buffer was too small. In that case,
	219	* ZSTD_compressStream2() expects a bigger buffer on the next call.
	220	* We can't provide a bigger buffer because we are limited with dest_size
	221	* which we pass to the ZSTD_compressStream2() at once.
	222	* So, we don't need any loops and just abort the compression when we
	223	* don't get 0 result on the first call.
	224	*/
	225	zstd_ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
	226
	227	if (zstd_ret) {
	228	if (zstd_ret > output.size - output.pos) {
	229	ret = -ENOMEM;
	230	} else {
	231	ret = -EIO;
	232	}
	233	goto out;
	234	}
	235
	236	/* make sure that zstd didn't overflow the dest buffer */
	237	assert(output.pos <= dest_size);
238	ret = output.pos;
239	out:
240	ZSTD_freeCCtx(cctx);
241	return ret;
242	}
243
244	/*
245	* qcow2_zstd_decompress()
246	*
247	* Decompress some data (not more than @src_size bytes) to produce exactly
248	* @dest_size bytes using zstd compression method
249	*
250	* @dest - destination buffer, @dest_size bytes
251	* @src - source buffer, @src_size bytes
252	*
253	* Returns: 0 on success
254	* -EIO on any error
255	*/
256	static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
257	const void *src, size_t src_size)
258	{
259	size_t zstd_ret = 0;
260	ssize_t ret = 0;
261	ZSTD_outBuffer output = {
262	.dst = dest,
263	.size = dest_size,
264	.pos = 0
265	};
266	ZSTD_inBuffer input = {
267	.src = src,
268	.size = src_size,
269	.pos = 0
270	};
271	ZSTD_DCtx *dctx = ZSTD_createDCtx();
272
273	if (!dctx) {
274	return -EIO;
275	}
276
277	/*
278	* The compressed stream from the input buffer may consist of more
279	* than one zstd frame. So we iterate until we get a fully
280	* uncompressed cluster.
281	* From zstd docs related to ZSTD_decompressStream:
282	* "return : 0 when a frame is completely decoded and fully flushed"
283	* We suppose that this means: each time ZSTD_decompressStream reads
284	* only ONE full frame and returns 0 if and only if that frame
285	* is completely decoded and flushed. Only after returning 0,
286	* ZSTD_decompressStream reads another ONE full frame.
287	*/
288	while (output.pos < output.size) {
289	size_t last_in_pos = input.pos;
290	size_t last_out_pos = output.pos;
291	zstd_ret = ZSTD_decompressStream(dctx, &output, &input);
292
293	if (ZSTD_isError(zstd_ret)) {
294	ret = -EIO;
295	break;
296	}
297
298	/*
299	* The ZSTD manual is vague about what to do if it reads
300	* the buffer partially, and we don't want to get stuck
301	* in an infinite loop where ZSTD_decompressStream
302	* returns > 0 waiting for another input chunk. So, we add
303	* a check which ensures that the loop makes some progress
304	* on each step.
305	*/
306	if (last_in_pos >= input.pos &&
307	last_out_pos >= output.pos) {
308	ret = -EIO;
309	break;
310	}
311	}
312	/*
313	* Make sure that we have the frame fully flushed here
314	* if not, we somehow managed to get uncompressed cluster
315	* greater then the cluster size, possibly because of its
316	* damage.
317	*/
318	if (zstd_ret > 0) {
319	ret = -EIO;
320	}
321
322	ZSTD_freeDCtx(dctx);
323	assert(ret == 0 \|\| ret == -EIO);
324	return ret;
325	}
326	#endif
327
56e2f1d8 VSO	328	static int qcow2_compress_pool_func(void *opaque)
	329	{
	330	Qcow2CompressData *data = opaque;
	331
	332	data->ret = data->func(data->dest, data->dest_size,
	333	data->src, data->src_size);
	334
	335	return 0;
	336	}
	337
56e2f1d8 VSO	338	static ssize_t coroutine_fn
	339	qcow2_co_do_compress(BlockDriverState bs, void dest, size_t dest_size,
	340	const void *src, size_t src_size, Qcow2CompressFunc func)
	341	{
56e2f1d8 VSO	342	Qcow2CompressData arg = {
	343	.dest = dest,
	344	.dest_size = dest_size,
	345	.src = src,
	346	.src_size = src_size,
	347	.func = func,
	348	};
	349
6f13a316	350	qcow2_co_process(bs, qcow2_compress_pool_func, &arg);
56e2f1d8 VSO	351
	352	return arg.ret;
	353	}
	354
25dd077d DP	355	/*
	356	* qcow2_co_compress()
	357	*
	358	* Compress @src_size bytes of data using the compression
	359	* method defined by the image compression type
	360	*
	361	* @dest - destination buffer, @dest_size bytes
	362	* @src - source buffer, @src_size bytes
	363	*
	364	* Returns: compressed size on success
	365	* a negative error code on failure
	366	*/
56e2f1d8 VSO	367	ssize_t coroutine_fn
	368	qcow2_co_compress(BlockDriverState bs, void dest, size_t dest_size,
	369	const void *src, size_t src_size)
	370	{
25dd077d DP	371	BDRVQcow2State *s = bs->opaque;
	372	Qcow2CompressFunc fn;
	373
	374	switch (s->compression_type) {
	375	case QCOW2_COMPRESSION_TYPE_ZLIB:
	376	fn = qcow2_zlib_compress;
	377	break;
	378
d298ac10 DP	379	#ifdef CONFIG_ZSTD
	380	case QCOW2_COMPRESSION_TYPE_ZSTD:
	381	fn = qcow2_zstd_compress;
	382	break;
	383	#endif
25dd077d DP	384	default:
	385	abort();
	386	}
	387
	388	return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
56e2f1d8 VSO	389	}
56e2f1d8 VSO	390
25dd077d DP	391	/*
	392	* qcow2_co_decompress()
	393	*
	394	* Decompress some data (not more than @src_size bytes) to produce exactly
	395	* @dest_size bytes using the compression method defined by the image
	396	* compression type
	397	*
	398	* @dest - destination buffer, @dest_size bytes
	399	* @src - source buffer, @src_size bytes
	400	*
	401	* Returns: 0 on success
	402	* a negative error code on failure
	403	*/
56e2f1d8 VSO	404	ssize_t coroutine_fn
	405	qcow2_co_decompress(BlockDriverState bs, void dest, size_t dest_size,
	406	const void *src, size_t src_size)
	407	{
25dd077d DP	408	BDRVQcow2State *s = bs->opaque;
	409	Qcow2CompressFunc fn;
	410
	411	switch (s->compression_type) {
	412	case QCOW2_COMPRESSION_TYPE_ZLIB:
	413	fn = qcow2_zlib_decompress;
	414	break;
	415
d298ac10 DP	416	#ifdef CONFIG_ZSTD
	417	case QCOW2_COMPRESSION_TYPE_ZSTD:
	418	fn = qcow2_zstd_decompress;
	419	break;
	420	#endif
25dd077d DP	421	default:
	422	abort();
	423	}
	424
	425	return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
56e2f1d8	426	}
8ac0f15f VSO	427
	428
	429	/*
	430	* Cryptography
	431	*/
	432
	433	/*
	434	* Qcow2EncDecFunc: common prototype of qcrypto_block_encrypt() and
	435	* qcrypto_block_decrypt() functions.
	436	*/
	437	typedef int (Qcow2EncDecFunc)(QCryptoBlock block, uint64_t offset,
	438	uint8_t buf, size_t len, Error *errp);
	439
	440	typedef struct Qcow2EncDecData {
	441	QCryptoBlock *block;
	442	uint64_t offset;
	443	uint8_t *buf;
	444	size_t len;
	445
	446	Qcow2EncDecFunc func;
	447	} Qcow2EncDecData;
	448
	449	static int qcow2_encdec_pool_func(void *opaque)
	450	{
	451	Qcow2EncDecData *data = opaque;
	452
	453	return data->func(data->block, data->offset, data->buf, data->len, NULL);
	454	}
	455
	456	static int coroutine_fn
603fbd07 ML	457	qcow2_co_encdec(BlockDriverState *bs, uint64_t host_offset,
	458	uint64_t guest_offset, void *buf, size_t len,
	459	Qcow2EncDecFunc func)
8ac0f15f VSO	460	{
	461	BDRVQcow2State *s = bs->opaque;
	462	Qcow2EncDecData arg = {
	463	.block = s->crypto,
603fbd07	464	.offset = s->crypt_physical_offset ? host_offset : guest_offset,
8ac0f15f VSO	465	.buf = buf,
	466	.len = len,
	467	.func = func,
	468	};
2d4b5256	469	uint64_t sector_size;
8ac0f15f	470
603fbd07 ML	471	assert(s->crypto);
603fbd07 ML	472
2d4b5256 AG	473	sector_size = qcrypto_block_get_sector_size(s->crypto);
	474	assert(QEMU_IS_ALIGNED(guest_offset, sector_size));
	475	assert(QEMU_IS_ALIGNED(host_offset, sector_size));
	476	assert(QEMU_IS_ALIGNED(len, sector_size));
	477
603fbd07	478	return len == 0 ? 0 : qcow2_co_process(bs, qcow2_encdec_pool_func, &arg);
8ac0f15f VSO	479	}
8ac0f15f VSO	480
603fbd07 ML	481	/*
	482	* qcow2_co_encrypt()
	483	*
	484	* Encrypts one or more contiguous aligned sectors
	485	*
	486	* @host_offset - underlying storage offset of the first sector of the
	487	* data to be encrypted
	488	*
	489	* @guest_offset - guest (virtual) offset of the first sector of the
	490	* data to be encrypted
	491	*
	492	* @buf - buffer with the data to encrypt, that after encryption
	493	* will be written to the underlying storage device at
	494	* @host_offset
	495	*
2d4b5256 AG	496	* @len - length of the buffer (must be a multiple of the encryption
2d4b5256 AG	497	* sector size)
603fbd07 ML	498	*
	499	* Depending on the encryption method, @host_offset and/or @guest_offset
	500	* may be used for generating the initialization vector for
	501	* encryption.
	502	*
	503	* Note that while the whole range must be aligned on sectors, it
	504	* does not have to be aligned on clusters and can also cross cluster
	505	* boundaries
	506	*/
8ac0f15f	507	int coroutine_fn
603fbd07 ML	508	qcow2_co_encrypt(BlockDriverState *bs, uint64_t host_offset,
603fbd07 ML	509	uint64_t guest_offset, void *buf, size_t len)
8ac0f15f	510	{
603fbd07 ML	511	return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,
603fbd07 ML	512	qcrypto_block_encrypt);
8ac0f15f VSO	513	}
8ac0f15f VSO	514
603fbd07 ML	515	/*
	516	* qcow2_co_decrypt()
	517	*
	518	* Decrypts one or more contiguous aligned sectors
	519	* Similar to qcow2_co_encrypt
	520	*/
8ac0f15f	521	int coroutine_fn
603fbd07 ML	522	qcow2_co_decrypt(BlockDriverState *bs, uint64_t host_offset,
603fbd07 ML	523	uint64_t guest_offset, void *buf, size_t len)
8ac0f15f	524	{
603fbd07 ML	525	return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,
603fbd07 ML	526	qcrypto_block_decrypt);
8ac0f15f	527	}