#include "qemu-common.h"
#include "qemu-barrier.h"
+#include "qemu-option.h"
+#include "qemu-config.h"
#include "sysemu.h"
#include "hw/hw.h"
+#include "hw/msi.h"
#include "gdbstub.h"
#include "kvm.h"
#include "bswap.h"
#include "memory.h"
#include "exec-memory.h"
+#include "event_notifier.h"
/* This check must be after config-host.h is included */
#ifdef CONFIG_EVENTFD
#include <sys/eventfd.h>
#endif
+#ifdef CONFIG_VALGRIND_H
+#include <valgrind/memcheck.h>
+#endif
+
/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
#define PAGE_SIZE TARGET_PAGE_SIZE
do { } while (0)
#endif
+#define KVM_MSI_HASHTAB_SIZE 256
+
typedef struct KVMSlot
{
target_phys_addr_t start_addr;
#ifdef KVM_CAP_SET_GUEST_DEBUG
struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
#endif
- int pit_in_kernel;
int pit_state2;
int xsave, xcrs;
int many_ioeventfds;
- int irqchip_inject_ioctl;
+ int intx_set_mask;
+ /* The man page (and posix) say ioctl numbers are signed int, but
+ * they're not. Linux, glibc and *BSD all treat ioctl numbers as
+ * unsigned, and treating them as signed here can break things */
+ unsigned irq_set_ioctl;
#ifdef KVM_CAP_IRQ_ROUTING
struct kvm_irq_routing *irq_routes;
int nr_allocated_irq_routes;
uint32_t *used_gsi_bitmap;
- unsigned int max_gsi;
+ unsigned int gsi_count;
+ QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
+ bool direct_msi;
#endif
};
KVMState *kvm_state;
bool kvm_kernel_irqchip;
+bool kvm_async_interrupts_allowed;
+bool kvm_irqfds_allowed;
+bool kvm_msi_via_irqfd_allowed;
+bool kvm_gsi_routing_allowed;
static const KVMCapabilityInfo kvm_required_capabilites[] = {
KVM_CAP_INFO(USER_MEMORY),
kvm_arch_reset_vcpu(env);
}
-int kvm_pit_in_kernel(void)
-{
- return kvm_state->pit_in_kernel;
-}
-
int kvm_init_vcpu(CPUArchState *env)
{
KVMState *s = kvm_state;
unsigned long page_number, c;
target_phys_addr_t addr, addr1;
unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
+ unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
do {
j = ffsl(c) - 1;
c &= ~(1ul << j);
- page_number = i * HOST_LONG_BITS + j;
+ page_number = (i * HOST_LONG_BITS + j) * hpratio;
addr1 = page_number * TARGET_PAGE_SIZE;
addr = section->offset_within_region + addr1;
- memory_region_set_dirty(section->mr, addr, TARGET_PAGE_SIZE);
+ memory_region_set_dirty(section->mr, addr,
+ TARGET_PAGE_SIZE * hpratio);
} while (c != 0);
}
}
{
int r;
- assert(match_data && section->size == 4);
+ assert(match_data && section->size <= 8);
- r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
- data, true);
+ r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
+ data, true, section->size);
if (r < 0) {
abort();
}
{
int r;
- r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
- data, false);
+ r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
+ data, false, section->size);
if (r < 0) {
abort();
}
static void kvm_eventfd_add(MemoryListener *listener,
MemoryRegionSection *section,
- bool match_data, uint64_t data, int fd)
+ bool match_data, uint64_t data,
+ EventNotifier *e)
{
if (section->address_space == get_system_memory()) {
- kvm_mem_ioeventfd_add(section, match_data, data, fd);
+ kvm_mem_ioeventfd_add(section, match_data, data,
+ event_notifier_get_fd(e));
} else {
- kvm_io_ioeventfd_add(section, match_data, data, fd);
+ kvm_io_ioeventfd_add(section, match_data, data,
+ event_notifier_get_fd(e));
}
}
static void kvm_eventfd_del(MemoryListener *listener,
MemoryRegionSection *section,
- bool match_data, uint64_t data, int fd)
+ bool match_data, uint64_t data,
+ EventNotifier *e)
{
if (section->address_space == get_system_memory()) {
- kvm_mem_ioeventfd_del(section, match_data, data, fd);
+ kvm_mem_ioeventfd_del(section, match_data, data,
+ event_notifier_get_fd(e));
} else {
- kvm_io_ioeventfd_del(section, match_data, data, fd);
+ kvm_io_ioeventfd_del(section, match_data, data,
+ event_notifier_get_fd(e));
}
}
}
}
-int kvm_irqchip_set_irq(KVMState *s, int irq, int level)
+int kvm_set_irq(KVMState *s, int irq, int level)
{
struct kvm_irq_level event;
int ret;
- assert(kvm_irqchip_in_kernel());
+ assert(kvm_async_interrupts_enabled());
event.level = level;
event.irq = irq;
- ret = kvm_vm_ioctl(s, s->irqchip_inject_ioctl, &event);
+ ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
if (ret < 0) {
- perror("kvm_set_irqchip_line");
+ perror("kvm_set_irq");
abort();
}
- return (s->irqchip_inject_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
+ return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
}
#ifdef KVM_CAP_IRQ_ROUTING
+typedef struct KVMMSIRoute {
+ struct kvm_irq_routing_entry kroute;
+ QTAILQ_ENTRY(KVMMSIRoute) entry;
+} KVMMSIRoute;
+
static void set_gsi(KVMState *s, unsigned int gsi)
{
- assert(gsi < s->max_gsi);
-
s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
}
+static void clear_gsi(KVMState *s, unsigned int gsi)
+{
+ s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
+}
+
static void kvm_init_irq_routing(KVMState *s)
{
- int gsi_count;
+ int gsi_count, i;
gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
if (gsi_count > 0) {
unsigned int gsi_bits, i;
/* Round up so we can search ints using ffs */
- gsi_bits = (gsi_count + 31) / 32;
+ gsi_bits = ALIGN(gsi_count, 32);
s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
- s->max_gsi = gsi_bits;
+ s->gsi_count = gsi_count;
/* Mark any over-allocated bits as already in use */
for (i = gsi_count; i < gsi_bits; i++) {
s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
s->nr_allocated_irq_routes = 0;
+ if (!s->direct_msi) {
+ for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
+ QTAILQ_INIT(&s->msi_hashtab[i]);
+ }
+ }
+
kvm_arch_init_irq_routing(s);
}
+static void kvm_irqchip_commit_routes(KVMState *s)
+{
+ int ret;
+
+ s->irq_routes->flags = 0;
+ ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
+ assert(ret == 0);
+}
+
static void kvm_add_routing_entry(KVMState *s,
struct kvm_irq_routing_entry *entry)
{
new->u = entry->u;
set_gsi(s, entry->gsi);
+
+ kvm_irqchip_commit_routes(s);
}
-void kvm_irqchip_add_route(KVMState *s, int irq, int irqchip, int pin)
+static int kvm_update_routing_entry(KVMState *s,
+ struct kvm_irq_routing_entry *new_entry)
+{
+ struct kvm_irq_routing_entry *entry;
+ int n;
+
+ for (n = 0; n < s->irq_routes->nr; n++) {
+ entry = &s->irq_routes->entries[n];
+ if (entry->gsi != new_entry->gsi) {
+ continue;
+ }
+
+ entry->type = new_entry->type;
+ entry->flags = new_entry->flags;
+ entry->u = new_entry->u;
+
+ kvm_irqchip_commit_routes(s);
+
+ return 0;
+ }
+
+ return -ESRCH;
+}
+
+void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
{
struct kvm_irq_routing_entry e;
+ assert(pin < s->gsi_count);
+
e.gsi = irq;
e.type = KVM_IRQ_ROUTING_IRQCHIP;
e.flags = 0;
kvm_add_routing_entry(s, &e);
}
-int kvm_irqchip_commit_routes(KVMState *s)
+void kvm_irqchip_release_virq(KVMState *s, int virq)
{
- s->irq_routes->flags = 0;
- return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
+ struct kvm_irq_routing_entry *e;
+ int i;
+
+ for (i = 0; i < s->irq_routes->nr; i++) {
+ e = &s->irq_routes->entries[i];
+ if (e->gsi == virq) {
+ s->irq_routes->nr--;
+ *e = s->irq_routes->entries[s->irq_routes->nr];
+ }
+ }
+ clear_gsi(s, virq);
+
+ kvm_irqchip_commit_routes(s);
+}
+
+static unsigned int kvm_hash_msi(uint32_t data)
+{
+ /* This is optimized for IA32 MSI layout. However, no other arch shall
+ * repeat the mistake of not providing a direct MSI injection API. */
+ return data & 0xff;
+}
+
+static void kvm_flush_dynamic_msi_routes(KVMState *s)
+{
+ KVMMSIRoute *route, *next;
+ unsigned int hash;
+
+ for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
+ QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
+ kvm_irqchip_release_virq(s, route->kroute.gsi);
+ QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
+ g_free(route);
+ }
+ }
+}
+
+static int kvm_irqchip_get_virq(KVMState *s)
+{
+ uint32_t *word = s->used_gsi_bitmap;
+ int max_words = ALIGN(s->gsi_count, 32) / 32;
+ int i, bit;
+ bool retry = true;
+
+again:
+ /* Return the lowest unused GSI in the bitmap */
+ for (i = 0; i < max_words; i++) {
+ bit = ffs(~word[i]);
+ if (!bit) {
+ continue;
+ }
+
+ return bit - 1 + i * 32;
+ }
+ if (!s->direct_msi && retry) {
+ retry = false;
+ kvm_flush_dynamic_msi_routes(s);
+ goto again;
+ }
+ return -ENOSPC;
+
+}
+
+static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
+{
+ unsigned int hash = kvm_hash_msi(msg.data);
+ KVMMSIRoute *route;
+
+ QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
+ if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
+ route->kroute.u.msi.address_hi == (msg.address >> 32) &&
+ route->kroute.u.msi.data == msg.data) {
+ return route;
+ }
+ }
+ return NULL;
+}
+
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
+{
+ struct kvm_msi msi;
+ KVMMSIRoute *route;
+
+ if (s->direct_msi) {
+ msi.address_lo = (uint32_t)msg.address;
+ msi.address_hi = msg.address >> 32;
+ msi.data = msg.data;
+ msi.flags = 0;
+ memset(msi.pad, 0, sizeof(msi.pad));
+
+ return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
+ }
+
+ route = kvm_lookup_msi_route(s, msg);
+ if (!route) {
+ int virq;
+
+ virq = kvm_irqchip_get_virq(s);
+ if (virq < 0) {
+ return virq;
+ }
+
+ route = g_malloc(sizeof(KVMMSIRoute));
+ route->kroute.gsi = virq;
+ route->kroute.type = KVM_IRQ_ROUTING_MSI;
+ route->kroute.flags = 0;
+ route->kroute.u.msi.address_lo = (uint32_t)msg.address;
+ route->kroute.u.msi.address_hi = msg.address >> 32;
+ route->kroute.u.msi.data = msg.data;
+
+ kvm_add_routing_entry(s, &route->kroute);
+
+ QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
+ entry);
+ }
+
+ assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
+
+ return kvm_set_irq(s, route->kroute.gsi, 1);
+}
+
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+{
+ struct kvm_irq_routing_entry kroute;
+ int virq;
+
+ if (!kvm_gsi_routing_enabled()) {
+ return -ENOSYS;
+ }
+
+ virq = kvm_irqchip_get_virq(s);
+ if (virq < 0) {
+ return virq;
+ }
+
+ kroute.gsi = virq;
+ kroute.type = KVM_IRQ_ROUTING_MSI;
+ kroute.flags = 0;
+ kroute.u.msi.address_lo = (uint32_t)msg.address;
+ kroute.u.msi.address_hi = msg.address >> 32;
+ kroute.u.msi.data = msg.data;
+
+ kvm_add_routing_entry(s, &kroute);
+
+ return virq;
+}
+
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+{
+ struct kvm_irq_routing_entry kroute;
+
+ if (!kvm_irqchip_in_kernel()) {
+ return -ENOSYS;
+ }
+
+ kroute.gsi = virq;
+ kroute.type = KVM_IRQ_ROUTING_MSI;
+ kroute.flags = 0;
+ kroute.u.msi.address_lo = (uint32_t)msg.address;
+ kroute.u.msi.address_hi = msg.address >> 32;
+ kroute.u.msi.data = msg.data;
+
+ return kvm_update_routing_entry(s, &kroute);
+}
+
+static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
+{
+ struct kvm_irqfd irqfd = {
+ .fd = fd,
+ .gsi = virq,
+ .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
+ };
+
+ if (!kvm_irqfds_enabled()) {
+ return -ENOSYS;
+ }
+
+ return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
}
#else /* !KVM_CAP_IRQ_ROUTING */
static void kvm_init_irq_routing(KVMState *s)
{
}
+
+void kvm_irqchip_release_virq(KVMState *s, int virq)
+{
+}
+
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
+{
+ abort();
+}
+
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+{
+ return -ENOSYS;
+}
+
+static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
+{
+ abort();
+}
#endif /* !KVM_CAP_IRQ_ROUTING */
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
+{
+ return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, true);
+}
+
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
+{
+ return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, false);
+}
+
static int kvm_irqchip_create(KVMState *s)
{
QemuOptsList *list = qemu_find_opts("machine");
if (QTAILQ_EMPTY(&list->head) ||
!qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
- "kernel_irqchip", false) ||
+ "kernel_irqchip", true) ||
!kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
return 0;
}
return ret;
}
- s->irqchip_inject_ioctl = KVM_IRQ_LINE;
- if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
- s->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
- }
kvm_kernel_irqchip = true;
+ /* If we have an in-kernel IRQ chip then we must have asynchronous
+ * interrupt delivery (though the reverse is not necessarily true)
+ */
+ kvm_async_interrupts_allowed = true;
kvm_init_irq_routing(s);
return 0;
}
+static int kvm_max_vcpus(KVMState *s)
+{
+ int ret;
+
+ /* Find number of supported CPUs using the recommended
+ * procedure from the kernel API documentation to cope with
+ * older kernels that may be missing capabilities.
+ */
+ ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
+ if (ret) {
+ return ret;
+ }
+ ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
+ if (ret) {
+ return ret;
+ }
+
+ return 4;
+}
+
int kvm_init(void)
{
static const char upgrade_note[] =
const KVMCapabilityInfo *missing_cap;
int ret;
int i;
+ int max_vcpus;
s = g_malloc0(sizeof(KVMState));
+ /*
+ * On systems where the kernel can support different base page
+ * sizes, host page size may be different from TARGET_PAGE_SIZE,
+ * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
+ * page size for the system though.
+ */
+ assert(TARGET_PAGE_SIZE <= getpagesize());
+
#ifdef KVM_CAP_SET_GUEST_DEBUG
QTAILQ_INIT(&s->kvm_sw_breakpoints);
#endif
goto err;
}
+ max_vcpus = kvm_max_vcpus(s);
+ if (smp_cpus > max_vcpus) {
+ ret = -EINVAL;
+ fprintf(stderr, "Number of SMP cpus requested (%d) exceeds max cpus "
+ "supported by KVM (%d)\n", smp_cpus, max_vcpus);
+ goto err;
+ }
+
s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
if (s->vmfd < 0) {
#ifdef TARGET_S390X
s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
#endif
+#ifdef KVM_CAP_IRQ_ROUTING
+ s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
+#endif
+
+ s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
+
+ s->irq_set_ioctl = KVM_IRQ_LINE;
+ if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
+ s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
+ }
+
ret = kvm_arch_init(s);
if (ret < 0) {
goto err;
return 0;
err:
- if (s) {
- if (s->vmfd >= 0) {
- close(s->vmfd);
- }
- if (s->fd != -1) {
- close(s->fd);
- }
+ if (s->vmfd >= 0) {
+ close(s->vmfd);
+ }
+ if (s->fd != -1) {
+ close(s->fd);
}
g_free(s);
qemu_mutex_lock_iothread();
kvm_arch_post_run(env, run);
- kvm_flush_coalesced_mmio_buffer();
-
if (run_ret < 0) {
if (run_ret == -EINTR || run_ret == -EAGAIN) {
DPRINTF("io window exit\n");
#endif
}
-int kvm_allows_irq0_override(void)
+int kvm_has_intx_set_mask(void)
+{
+ return kvm_state->intx_set_mask;
+}
+
+void *kvm_vmalloc(ram_addr_t size)
{
- return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
+#ifdef TARGET_S390X
+ void *mem;
+
+ mem = kvm_arch_vmalloc(size);
+ if (mem) {
+ return mem;
+ }
+#endif
+ return qemu_vmalloc(size);
}
void kvm_setup_guest_memory(void *start, size_t size)
{
+#ifdef CONFIG_VALGRIND_H
+ VALGRIND_MAKE_MEM_DEFINED(start, size);
+#endif
if (!kvm_has_sync_mmu()) {
int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
return r;
}
-int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
+int kvm_set_ioeventfd_mmio(int fd, uint32_t addr, uint32_t val, bool assign,
+ uint32_t size)
{
int ret;
struct kvm_ioeventfd iofd;
iofd.datamatch = val;
iofd.addr = addr;
- iofd.len = 4;
+ iofd.len = size;
iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
iofd.fd = fd;
projects / qemu.git / blobdiff