4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "hw/virtio/vhost.h"
31 /* Todo:need to add the multiqueue support here */
32 typedef struct VhostVDPAState {
34 struct vhost_vdpa vhost_vdpa;
35 VHostNetState *vhost_net;
37 /* Control commands shadow buffers */
38 void *cvq_cmd_out_buffer, *cvq_cmd_in_buffer;
42 const int vdpa_feature_bits[] = {
43 VIRTIO_F_NOTIFY_ON_EMPTY,
44 VIRTIO_RING_F_INDIRECT_DESC,
45 VIRTIO_RING_F_EVENT_IDX,
49 VIRTIO_NET_F_GUEST_CSUM,
51 VIRTIO_NET_F_GUEST_TSO4,
52 VIRTIO_NET_F_GUEST_TSO6,
53 VIRTIO_NET_F_GUEST_ECN,
54 VIRTIO_NET_F_GUEST_UFO,
55 VIRTIO_NET_F_HOST_TSO4,
56 VIRTIO_NET_F_HOST_TSO6,
57 VIRTIO_NET_F_HOST_ECN,
58 VIRTIO_NET_F_HOST_UFO,
59 VIRTIO_NET_F_MRG_RXBUF,
62 VIRTIO_NET_F_CTRL_RX_EXTRA,
63 VIRTIO_NET_F_CTRL_VLAN,
64 VIRTIO_NET_F_GUEST_ANNOUNCE,
65 VIRTIO_NET_F_CTRL_MAC_ADDR,
69 VIRTIO_F_IOMMU_PLATFORM,
72 VIRTIO_NET_F_HASH_REPORT,
73 VIRTIO_NET_F_GUEST_ANNOUNCE,
75 VHOST_INVALID_FEATURE_BIT
78 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
80 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
81 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
85 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
89 struct vhost_dev *hdev;
91 hdev = (struct vhost_dev *)&net->dev;
92 ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
93 if (device_id != VIRTIO_ID_NET) {
99 static int vhost_vdpa_add(NetClientState *ncs, void *be,
100 int queue_pair_index, int nvqs)
102 VhostNetOptions options;
103 struct vhost_net *net = NULL;
107 options.backend_type = VHOST_BACKEND_TYPE_VDPA;
108 assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
109 s = DO_UPCAST(VhostVDPAState, nc, ncs);
110 options.net_backend = ncs;
112 options.busyloop_timeout = 0;
115 net = vhost_net_init(&options);
117 error_report("failed to init vhost_net for queue");
121 ret = vhost_vdpa_net_check_device_id(net);
127 vhost_net_cleanup(net);
133 static void vhost_vdpa_cleanup(NetClientState *nc)
135 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
137 qemu_vfree(s->cvq_cmd_out_buffer);
138 qemu_vfree(s->cvq_cmd_in_buffer);
140 vhost_net_cleanup(s->vhost_net);
141 g_free(s->vhost_net);
144 if (s->vhost_vdpa.device_fd >= 0) {
145 qemu_close(s->vhost_vdpa.device_fd);
146 s->vhost_vdpa.device_fd = -1;
150 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
152 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
157 static bool vhost_vdpa_has_ufo(NetClientState *nc)
159 assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
160 VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
161 uint64_t features = 0;
162 features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
163 features = vhost_net_get_features(s->vhost_net, features);
164 return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
168 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
171 const char *driver = object_class_get_name(oc);
173 if (!g_str_has_prefix(driver, "virtio-net-")) {
174 error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
181 /** Dummy receive in case qemu falls back to userland tap networking */
182 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
188 static NetClientInfo net_vhost_vdpa_info = {
189 .type = NET_CLIENT_DRIVER_VHOST_VDPA,
190 .size = sizeof(VhostVDPAState),
191 .receive = vhost_vdpa_receive,
192 .cleanup = vhost_vdpa_cleanup,
193 .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
194 .has_ufo = vhost_vdpa_has_ufo,
195 .check_peer_type = vhost_vdpa_check_peer_type,
198 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
200 VhostIOVATree *tree = v->iova_tree;
203 * No need to specify size or to look for more translations since
204 * this contiguous chunk was allocated by us.
206 .translated_addr = (hwaddr)(uintptr_t)addr,
208 const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
211 if (unlikely(!map)) {
212 error_report("Cannot locate expected map");
216 r = vhost_vdpa_dma_unmap(v, map->iova, map->size + 1);
217 if (unlikely(r != 0)) {
218 error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
221 vhost_iova_tree_remove(tree, map);
224 static size_t vhost_vdpa_net_cvq_cmd_len(void)
227 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
228 * In buffer is always 1 byte, so it should fit here
230 return sizeof(struct virtio_net_ctrl_hdr) +
231 2 * sizeof(struct virtio_net_ctrl_mac) +
232 MAC_TABLE_ENTRIES * ETH_ALEN;
235 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
237 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
240 /** Copy and map a guest buffer. */
241 static bool vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v,
242 const struct iovec *out_data,
243 size_t out_num, size_t data_len, void *buf,
244 size_t *written, bool write)
249 if (unlikely(!data_len)) {
250 qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid legnth of %s buffer\n",
251 __func__, write ? "in" : "out");
255 *written = iov_to_buf(out_data, out_num, 0, buf, data_len);
256 map.translated_addr = (hwaddr)(uintptr_t)buf;
257 map.size = vhost_vdpa_net_cvq_cmd_page_len() - 1;
258 map.perm = write ? IOMMU_RW : IOMMU_RO,
259 r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
260 if (unlikely(r != IOVA_OK)) {
261 error_report("Cannot map injected element");
265 r = vhost_vdpa_dma_map(v, map.iova, vhost_vdpa_net_cvq_cmd_page_len(), buf,
267 if (unlikely(r < 0)) {
274 vhost_iova_tree_remove(v->iova_tree, &map);
279 * Copy the guest element into a dedicated buffer suitable to be sent to NIC
281 * @iov: [0] is the out buffer, [1] is the in one
283 static bool vhost_vdpa_net_cvq_map_elem(VhostVDPAState *s,
284 VirtQueueElement *elem,
290 iov[0].iov_base = s->cvq_cmd_out_buffer;
291 ok = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, elem->out_sg, elem->out_num,
292 vhost_vdpa_net_cvq_cmd_len(), iov[0].iov_base,
293 &iov[0].iov_len, false);
298 iov[1].iov_base = s->cvq_cmd_in_buffer;
299 ok = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, NULL, 0,
300 sizeof(virtio_net_ctrl_ack), iov[1].iov_base,
303 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
307 iov[1].iov_len = sizeof(virtio_net_ctrl_ack);
312 * Do not forward commands not supported by SVQ. Otherwise, the device could
313 * accept it and qemu would not know how to update the device model.
315 static bool vhost_vdpa_net_cvq_validate_cmd(const struct iovec *out,
318 struct virtio_net_ctrl_hdr ctrl;
321 n = iov_to_buf(out, out_num, 0, &ctrl, sizeof(ctrl));
322 if (unlikely(n < sizeof(ctrl))) {
323 qemu_log_mask(LOG_GUEST_ERROR,
324 "%s: invalid legnth of out buffer %zu\n", __func__, n);
328 switch (ctrl.class) {
329 case VIRTIO_NET_CTRL_MAC:
331 case VIRTIO_NET_CTRL_MAC_ADDR_SET:
334 qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid mac cmd %u\n",
339 qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid control class %u\n",
340 __func__, ctrl.class);
347 * Validate and copy control virtqueue commands.
349 * Following QEMU guidelines, we offer a copy of the buffers to the device to
350 * prevent TOCTOU bugs.
352 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
353 VirtQueueElement *elem,
356 VhostVDPAState *s = opaque;
357 size_t in_len, dev_written;
358 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
359 /* out and in buffers sent to the device */
360 struct iovec dev_buffers[2] = {
361 { .iov_base = s->cvq_cmd_out_buffer },
362 { .iov_base = s->cvq_cmd_in_buffer },
364 /* in buffer used for device model */
365 const struct iovec in = {
367 .iov_len = sizeof(status),
372 ok = vhost_vdpa_net_cvq_map_elem(s, elem, dev_buffers);
377 ok = vhost_vdpa_net_cvq_validate_cmd(&dev_buffers[0], 1);
382 r = vhost_svq_add(svq, &dev_buffers[0], 1, &dev_buffers[1], 1, elem);
383 if (unlikely(r != 0)) {
384 if (unlikely(r == -ENOSPC)) {
385 qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
392 * We can poll here since we've had BQL from the time we sent the
393 * descriptor. Also, we need to take the answer before SVQ pulls by itself,
394 * when BQL is released
396 dev_written = vhost_svq_poll(svq);
397 if (unlikely(dev_written < sizeof(status))) {
398 error_report("Insufficient written data (%zu)", dev_written);
402 memcpy(&status, dev_buffers[1].iov_base, sizeof(status));
403 if (status != VIRTIO_NET_OK) {
407 status = VIRTIO_NET_ERR;
408 virtio_net_handle_ctrl_iov(svq->vdev, &in, 1, dev_buffers, 1);
409 if (status != VIRTIO_NET_OK) {
410 error_report("Bad CVQ processing in model");
414 in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
416 if (unlikely(in_len < sizeof(status))) {
417 error_report("Bad device CVQ written length");
419 vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
421 if (dev_buffers[0].iov_base) {
422 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, dev_buffers[0].iov_base);
424 if (dev_buffers[1].iov_base) {
425 vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, dev_buffers[1].iov_base);
430 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
431 .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
434 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
438 int queue_pair_index,
442 NetClientState *nc = NULL;
447 nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
450 nc = qemu_new_net_control_client(&net_vhost_vdpa_info, peer,
453 snprintf(nc->info_str, sizeof(nc->info_str), TYPE_VHOST_VDPA);
454 s = DO_UPCAST(VhostVDPAState, nc, nc);
456 s->vhost_vdpa.device_fd = vdpa_device_fd;
457 s->vhost_vdpa.index = queue_pair_index;
459 s->cvq_cmd_out_buffer = qemu_memalign(qemu_real_host_page_size(),
460 vhost_vdpa_net_cvq_cmd_page_len());
461 memset(s->cvq_cmd_out_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len());
462 s->cvq_cmd_in_buffer = qemu_memalign(qemu_real_host_page_size(),
463 vhost_vdpa_net_cvq_cmd_page_len());
464 memset(s->cvq_cmd_in_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len());
466 s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
467 s->vhost_vdpa.shadow_vq_ops_opaque = s;
469 ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
471 qemu_del_net_client(nc);
477 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
479 int ret = ioctl(fd, VHOST_GET_FEATURES, features);
480 if (unlikely(ret < 0)) {
481 error_setg_errno(errp, errno,
482 "Fail to query features from vhost-vDPA device");
487 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
488 int *has_cvq, Error **errp)
490 unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
491 g_autofree struct vhost_vdpa_config *config = NULL;
492 __virtio16 *max_queue_pairs;
495 if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
501 if (features & (1 << VIRTIO_NET_F_MQ)) {
502 config = g_malloc0(config_size + sizeof(*max_queue_pairs));
503 config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
504 config->len = sizeof(*max_queue_pairs);
506 ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
508 error_setg(errp, "Fail to get config from vhost-vDPA device");
512 max_queue_pairs = (__virtio16 *)&config->buf;
514 return lduw_le_p(max_queue_pairs);
520 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
521 NetClientState *peer, Error **errp)
523 const NetdevVhostVDPAOptions *opts;
526 g_autofree NetClientState **ncs = NULL;
528 int queue_pairs, r, i, has_cvq = 0;
530 assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
531 opts = &netdev->u.vhost_vdpa;
532 if (!opts->vhostdev) {
533 error_setg(errp, "vdpa character device not specified with vhostdev");
537 vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
538 if (vdpa_device_fd == -1) {
542 r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
543 if (unlikely(r < 0)) {
547 queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
549 if (queue_pairs < 0) {
550 qemu_close(vdpa_device_fd);
554 ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
556 for (i = 0; i < queue_pairs; i++) {
557 ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
558 vdpa_device_fd, i, 2, true);
564 nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
565 vdpa_device_fd, i, 1, false);
574 for (i--; i >= 0; i--) {
575 qemu_del_net_client(ncs[i]);
578 qemu_close(vdpa_device_fd);