2 * QEMU Block driver for CURL images
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 #include "qemu-common.h"
25 #include "block/block_int.h"
26 #include <curl/curl.h>
29 // #define DEBUG_VERBOSE
32 #define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
34 #define DPRINTF(fmt, ...) do { } while (0)
37 #if LIBCURL_VERSION_NUM >= 0x071000
38 /* The multi interface timer callback was introduced in 7.16.0 */
39 #define NEED_CURL_TIMER_CALLBACK
42 #define PROTOCOLS (CURLPROTO_HTTP | CURLPROTO_HTTPS | \
43 CURLPROTO_FTP | CURLPROTO_FTPS | \
46 #define CURL_NUM_STATES 8
47 #define CURL_NUM_ACB 8
48 #define SECTOR_SIZE 512
49 #define READ_AHEAD_SIZE (256 * 1024)
51 #define FIND_RET_NONE 0
53 #define FIND_RET_WAIT 2
57 typedef struct CURLAIOCB {
58 BlockDriverAIOCB common;
69 typedef struct CURLState
71 struct BDRVCURLState *s;
72 CURLAIOCB *acb[CURL_NUM_ACB];
79 char errmsg[CURL_ERROR_SIZE];
83 typedef struct BDRVCURLState {
87 CURLState states[CURL_NUM_STATES];
89 size_t readahead_size;
93 static void curl_clean_state(CURLState *s);
94 static void curl_multi_do(void *arg);
96 #ifdef NEED_CURL_TIMER_CALLBACK
97 static int curl_timer_cb(CURLM *multi, long timeout_ms, void *opaque)
99 BDRVCURLState *s = opaque;
101 DPRINTF("CURL: timer callback timeout_ms %ld\n", timeout_ms);
102 if (timeout_ms == -1) {
103 timer_del(&s->timer);
105 int64_t timeout_ns = (int64_t)timeout_ms * 1000 * 1000;
107 qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ns);
113 static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
116 DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
119 qemu_aio_set_fd_handler(fd, curl_multi_do, NULL, s);
122 qemu_aio_set_fd_handler(fd, NULL, curl_multi_do, s);
124 case CURL_POLL_INOUT:
125 qemu_aio_set_fd_handler(fd, curl_multi_do, curl_multi_do, s);
127 case CURL_POLL_REMOVE:
128 qemu_aio_set_fd_handler(fd, NULL, NULL, NULL);
135 static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
137 BDRVCURLState *s = opaque;
138 size_t realsize = size * nmemb;
139 const char *accept_line = "Accept-Ranges: bytes";
141 if (realsize >= strlen(accept_line)
142 && strncmp((char *)ptr, accept_line, strlen(accept_line)) == 0) {
143 s->accept_range = true;
149 static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
151 CURLState *s = ((CURLState*)opaque);
152 size_t realsize = size * nmemb;
155 DPRINTF("CURL: Just reading %zd bytes\n", realsize);
157 if (!s || !s->orig_buf)
160 if (s->buf_off >= s->buf_len) {
161 /* buffer full, read nothing */
164 realsize = MIN(realsize, s->buf_len - s->buf_off);
165 memcpy(s->orig_buf + s->buf_off, ptr, realsize);
166 s->buf_off += realsize;
168 for(i=0; i<CURL_NUM_ACB; i++) {
169 CURLAIOCB *acb = s->acb[i];
174 if ((s->buf_off >= acb->end)) {
175 qemu_iovec_from_buf(acb->qiov, 0, s->orig_buf + acb->start,
176 acb->end - acb->start);
177 acb->common.cb(acb->common.opaque, 0);
178 qemu_aio_release(acb);
186 static int curl_find_buf(BDRVCURLState *s, size_t start, size_t len,
190 size_t end = start + len;
192 for (i=0; i<CURL_NUM_STATES; i++) {
193 CURLState *state = &s->states[i];
194 size_t buf_end = (state->buf_start + state->buf_off);
195 size_t buf_fend = (state->buf_start + state->buf_len);
197 if (!state->orig_buf)
202 // Does the existing buffer cover our section?
203 if ((start >= state->buf_start) &&
204 (start <= buf_end) &&
205 (end >= state->buf_start) &&
208 char *buf = state->orig_buf + (start - state->buf_start);
210 qemu_iovec_from_buf(acb->qiov, 0, buf, len);
211 acb->common.cb(acb->common.opaque, 0);
216 // Wait for unfinished chunks
217 if ((start >= state->buf_start) &&
218 (start <= buf_fend) &&
219 (end >= state->buf_start) &&
224 acb->start = start - state->buf_start;
225 acb->end = acb->start + len;
227 for (j=0; j<CURL_NUM_ACB; j++) {
228 if (!state->acb[j]) {
230 return FIND_RET_WAIT;
236 return FIND_RET_NONE;
239 static void curl_multi_read(BDRVCURLState *s)
243 /* Try to find done transfers, so we can free the easy
247 msg = curl_multi_info_read(s->multi, &msgs_in_queue);
251 if (msg->msg == CURLMSG_NONE)
257 CURLState *state = NULL;
258 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE,
261 /* ACBs for successful messages get completed in curl_read_cb */
262 if (msg->data.result != CURLE_OK) {
264 for (i = 0; i < CURL_NUM_ACB; i++) {
265 CURLAIOCB *acb = state->acb[i];
271 acb->common.cb(acb->common.opaque, -EIO);
272 qemu_aio_release(acb);
273 state->acb[i] = NULL;
277 curl_clean_state(state);
284 } while(msgs_in_queue);
287 static void curl_multi_do(void *arg)
289 BDRVCURLState *s = (BDRVCURLState *)arg;
298 r = curl_multi_socket_all(s->multi, &running);
299 } while(r == CURLM_CALL_MULTI_PERFORM);
304 static void curl_multi_timeout_do(void *arg)
306 #ifdef NEED_CURL_TIMER_CALLBACK
307 BDRVCURLState *s = (BDRVCURLState *)arg;
314 curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0, &running);
322 static CURLState *curl_init_state(BDRVCURLState *s)
324 CURLState *state = NULL;
328 for (i=0; i<CURL_NUM_STATES; i++) {
329 for (j=0; j<CURL_NUM_ACB; j++)
330 if (s->states[i].acb[j])
332 if (s->states[i].in_use)
335 state = &s->states[i];
346 state->curl = curl_easy_init();
350 curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
351 curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
352 curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION,
353 (void *)curl_read_cb);
354 curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
355 curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
356 curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
357 curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
358 curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
359 curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
360 curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
362 /* Restrict supported protocols to avoid security issues in the more
363 * obscure protocols. For example, do not allow POP3/SMTP/IMAP see
366 * Restricting protocols is only supported from 7.19.4 upwards.
368 #if LIBCURL_VERSION_NUM >= 0x071304
369 curl_easy_setopt(state->curl, CURLOPT_PROTOCOLS, PROTOCOLS);
370 curl_easy_setopt(state->curl, CURLOPT_REDIR_PROTOCOLS, PROTOCOLS);
374 curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
383 static void curl_clean_state(CURLState *s)
386 curl_multi_remove_handle(s->s->multi, s->curl);
390 static void curl_parse_filename(const char *filename, QDict *options,
394 #define RA_OPTSTR ":readahead="
400 file = g_strdup(filename);
402 /* Parse a trailing ":readahead=#:" param, if present. */
403 ra = file + strlen(file) - 1;
405 if (parse_state == 0) {
411 } else if (parse_state == 1) {
412 if (*ra > '9' || *ra < '0') {
413 char *opt_start = ra - strlen(RA_OPTSTR) + 1;
414 if (opt_start > file &&
415 strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
417 ra -= strlen(RA_OPTSTR) - 1;
419 qdict_put(options, "readahead", qstring_from_str(ra_val));
427 qdict_put(options, "url", qstring_from_str(file));
432 static QemuOptsList runtime_opts = {
434 .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
438 .type = QEMU_OPT_STRING,
439 .help = "URL to open",
443 .type = QEMU_OPT_SIZE,
444 .help = "Readahead size",
446 { /* end of list */ }
450 static int curl_open(BlockDriverState *bs, QDict *options, int flags,
453 BDRVCURLState *s = bs->opaque;
454 CURLState *state = NULL;
456 Error *local_err = NULL;
460 static int inited = 0;
462 if (flags & BDRV_O_RDWR) {
463 error_setg(errp, "curl block device does not support writes");
467 opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
468 qemu_opts_absorb_qdict(opts, options, &local_err);
470 error_propagate(errp, local_err);
474 s->readahead_size = qemu_opt_get_size(opts, "readahead", READ_AHEAD_SIZE);
475 if ((s->readahead_size & 0x1ff) != 0) {
476 error_setg(errp, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512",
481 file = qemu_opt_get(opts, "url");
483 error_setg(errp, "curl block driver requires an 'url' option");
488 curl_global_init(CURL_GLOBAL_ALL);
492 DPRINTF("CURL: Opening %s\n", file);
493 s->url = g_strdup(file);
494 state = curl_init_state(s);
500 s->accept_range = false;
501 curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
502 curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION,
504 curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, s);
505 if (curl_easy_perform(state->curl))
507 curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
512 if ((!strncasecmp(s->url, "http://", strlen("http://"))
513 || !strncasecmp(s->url, "https://", strlen("https://")))
514 && !s->accept_range) {
515 pstrcpy(state->errmsg, CURL_ERROR_SIZE,
516 "Server does not support 'range' (byte ranges).");
519 DPRINTF("CURL: Size = %zd\n", s->len);
521 curl_clean_state(state);
522 curl_easy_cleanup(state->curl);
525 aio_timer_init(bdrv_get_aio_context(bs), &s->timer,
526 QEMU_CLOCK_REALTIME, SCALE_NS,
527 curl_multi_timeout_do, s);
529 // Now we know the file exists and its size, so let's
530 // initialize the multi interface!
532 s->multi = curl_multi_init();
533 curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, s);
534 curl_multi_setopt(s->multi, CURLMOPT_SOCKETFUNCTION, curl_sock_cb);
535 #ifdef NEED_CURL_TIMER_CALLBACK
536 curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, s);
537 curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, curl_timer_cb);
545 error_setg(errp, "CURL: Error opening file: %s", state->errmsg);
546 curl_easy_cleanup(state->curl);
554 static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
556 // Do we have to implement canceling? Seems to work without...
559 static const AIOCBInfo curl_aiocb_info = {
560 .aiocb_size = sizeof(CURLAIOCB),
561 .cancel = curl_aio_cancel,
565 static void curl_readv_bh_cb(void *p)
570 BDRVCURLState *s = acb->common.bs->opaque;
572 qemu_bh_delete(acb->bh);
575 size_t start = acb->sector_num * SECTOR_SIZE;
578 // In case we have the requested data already (e.g. read-ahead),
579 // we can just call the callback and be done.
580 switch (curl_find_buf(s, start, acb->nb_sectors * SECTOR_SIZE, acb)) {
582 qemu_aio_release(acb);
590 // No cache found, so let's start a new request
591 state = curl_init_state(s);
593 acb->common.cb(acb->common.opaque, -EIO);
594 qemu_aio_release(acb);
599 acb->end = (acb->nb_sectors * SECTOR_SIZE);
603 g_free(state->orig_buf);
604 state->buf_start = start;
605 state->buf_len = acb->end + s->readahead_size;
606 end = MIN(start + state->buf_len, s->len) - 1;
607 state->orig_buf = g_malloc(state->buf_len);
610 snprintf(state->range, 127, "%zd-%zd", start, end);
611 DPRINTF("CURL (AIO): Reading %d at %zd (%s)\n",
612 (acb->nb_sectors * SECTOR_SIZE), start, state->range);
613 curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
615 curl_multi_add_handle(s->multi, state->curl);
620 static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
621 int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
622 BlockDriverCompletionFunc *cb, void *opaque)
626 acb = qemu_aio_get(&curl_aiocb_info, bs, cb, opaque);
629 acb->sector_num = sector_num;
630 acb->nb_sectors = nb_sectors;
632 acb->bh = qemu_bh_new(curl_readv_bh_cb, acb);
633 qemu_bh_schedule(acb->bh);
637 static void curl_close(BlockDriverState *bs)
639 BDRVCURLState *s = bs->opaque;
642 DPRINTF("CURL: Close\n");
643 for (i=0; i<CURL_NUM_STATES; i++) {
644 if (s->states[i].in_use)
645 curl_clean_state(&s->states[i]);
646 if (s->states[i].curl) {
647 curl_easy_cleanup(s->states[i].curl);
648 s->states[i].curl = NULL;
650 if (s->states[i].orig_buf) {
651 g_free(s->states[i].orig_buf);
652 s->states[i].orig_buf = NULL;
656 curl_multi_cleanup(s->multi);
658 timer_del(&s->timer);
663 static int64_t curl_getlength(BlockDriverState *bs)
665 BDRVCURLState *s = bs->opaque;
669 static BlockDriver bdrv_http = {
670 .format_name = "http",
671 .protocol_name = "http",
673 .instance_size = sizeof(BDRVCURLState),
674 .bdrv_parse_filename = curl_parse_filename,
675 .bdrv_file_open = curl_open,
676 .bdrv_close = curl_close,
677 .bdrv_getlength = curl_getlength,
679 .bdrv_aio_readv = curl_aio_readv,
682 static BlockDriver bdrv_https = {
683 .format_name = "https",
684 .protocol_name = "https",
686 .instance_size = sizeof(BDRVCURLState),
687 .bdrv_parse_filename = curl_parse_filename,
688 .bdrv_file_open = curl_open,
689 .bdrv_close = curl_close,
690 .bdrv_getlength = curl_getlength,
692 .bdrv_aio_readv = curl_aio_readv,
695 static BlockDriver bdrv_ftp = {
696 .format_name = "ftp",
697 .protocol_name = "ftp",
699 .instance_size = sizeof(BDRVCURLState),
700 .bdrv_parse_filename = curl_parse_filename,
701 .bdrv_file_open = curl_open,
702 .bdrv_close = curl_close,
703 .bdrv_getlength = curl_getlength,
705 .bdrv_aio_readv = curl_aio_readv,
708 static BlockDriver bdrv_ftps = {
709 .format_name = "ftps",
710 .protocol_name = "ftps",
712 .instance_size = sizeof(BDRVCURLState),
713 .bdrv_parse_filename = curl_parse_filename,
714 .bdrv_file_open = curl_open,
715 .bdrv_close = curl_close,
716 .bdrv_getlength = curl_getlength,
718 .bdrv_aio_readv = curl_aio_readv,
721 static BlockDriver bdrv_tftp = {
722 .format_name = "tftp",
723 .protocol_name = "tftp",
725 .instance_size = sizeof(BDRVCURLState),
726 .bdrv_parse_filename = curl_parse_filename,
727 .bdrv_file_open = curl_open,
728 .bdrv_close = curl_close,
729 .bdrv_getlength = curl_getlength,
731 .bdrv_aio_readv = curl_aio_readv,
734 static void curl_block_init(void)
736 bdrv_register(&bdrv_http);
737 bdrv_register(&bdrv_https);
738 bdrv_register(&bdrv_ftp);
739 bdrv_register(&bdrv_ftps);
740 bdrv_register(&bdrv_tftp);
743 block_init(curl_block_init);