#include "qemu-common.h"
#include "qemu-barrier.h"
+#include "qemu-option.h"
+#include "qemu-config.h"
#include "sysemu.h"
#include "hw/hw.h"
+#include "hw/msi.h"
#include "gdbstub.h"
#include "kvm.h"
#include "bswap.h"
#include <sys/eventfd.h>
#endif
-/* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
+/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
#define PAGE_SIZE TARGET_PAGE_SIZE
//#define DEBUG_KVM
do { } while (0)
#endif
+#define KVM_MSI_HASHTAB_SIZE 256
+
typedef struct KVMSlot
{
target_phys_addr_t start_addr;
#ifdef KVM_CAP_SET_GUEST_DEBUG
struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
#endif
- int pit_in_kernel;
int pit_state2;
int xsave, xcrs;
int many_ioeventfds;
- int irqchip_inject_ioctl;
+ /* The man page (and posix) say ioctl numbers are signed int, but
+ * they're not. Linux, glibc and *BSD all treat ioctl numbers as
+ * unsigned, and treating them as signed here can break things */
+ unsigned irqchip_inject_ioctl;
#ifdef KVM_CAP_IRQ_ROUTING
struct kvm_irq_routing *irq_routes;
int nr_allocated_irq_routes;
uint32_t *used_gsi_bitmap;
- unsigned int max_gsi;
+ unsigned int gsi_count;
+ QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
+ bool direct_msi;
#endif
};
static void kvm_reset_vcpu(void *opaque)
{
- CPUState *env = opaque;
+ CPUArchState *env = opaque;
kvm_arch_reset_vcpu(env);
}
-int kvm_pit_in_kernel(void)
-{
- return kvm_state->pit_in_kernel;
-}
-
-int kvm_init_vcpu(CPUState *env)
+int kvm_init_vcpu(CPUArchState *env)
{
KVMState *s = kvm_state;
long mmap_size;
unsigned long page_number, c;
target_phys_addr_t addr, addr1;
unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
+ unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
do {
j = ffsl(c) - 1;
c &= ~(1ul << j);
- page_number = i * HOST_LONG_BITS + j;
+ page_number = (i * HOST_LONG_BITS + j) * hpratio;
addr1 = page_number * TARGET_PAGE_SIZE;
addr = section->offset_within_region + addr1;
- memory_region_set_dirty(section->mr, addr, TARGET_PAGE_SIZE);
+ memory_region_set_dirty(section->mr, addr,
+ TARGET_PAGE_SIZE * hpratio);
} while (c != 0);
}
}
{
int r;
- assert(match_data && section->size == 4);
+ assert(match_data && section->size <= 8);
- r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
- data, true);
+ r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
+ data, true, section->size);
if (r < 0) {
abort();
}
{
int r;
- r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
- data, false);
+ r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
+ data, false, section->size);
if (r < 0) {
abort();
}
.priority = 10,
};
-static void kvm_handle_interrupt(CPUState *env, int mask)
+static void kvm_handle_interrupt(CPUArchState *env, int mask)
{
env->interrupt_request |= mask;
}
#ifdef KVM_CAP_IRQ_ROUTING
+typedef struct KVMMSIRoute {
+ struct kvm_irq_routing_entry kroute;
+ QTAILQ_ENTRY(KVMMSIRoute) entry;
+} KVMMSIRoute;
+
static void set_gsi(KVMState *s, unsigned int gsi)
{
- assert(gsi < s->max_gsi);
-
s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
}
+static void clear_gsi(KVMState *s, unsigned int gsi)
+{
+ s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
+}
+
static void kvm_init_irq_routing(KVMState *s)
{
- int gsi_count;
+ int gsi_count, i;
gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
if (gsi_count > 0) {
unsigned int gsi_bits, i;
/* Round up so we can search ints using ffs */
- gsi_bits = (gsi_count + 31) / 32;
+ gsi_bits = ALIGN(gsi_count, 32);
s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
- s->max_gsi = gsi_bits;
+ s->gsi_count = gsi_count;
/* Mark any over-allocated bits as already in use */
for (i = gsi_count; i < gsi_bits; i++) {
s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
s->nr_allocated_irq_routes = 0;
+ if (!s->direct_msi) {
+ for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
+ QTAILQ_INIT(&s->msi_hashtab[i]);
+ }
+ }
+
kvm_arch_init_irq_routing(s);
}
+static void kvm_irqchip_commit_routes(KVMState *s)
+{
+ int ret;
+
+ s->irq_routes->flags = 0;
+ ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
+ assert(ret == 0);
+}
+
static void kvm_add_routing_entry(KVMState *s,
struct kvm_irq_routing_entry *entry)
{
new->u = entry->u;
set_gsi(s, entry->gsi);
+
+ kvm_irqchip_commit_routes(s);
}
-void kvm_irqchip_add_route(KVMState *s, int irq, int irqchip, int pin)
+void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
{
struct kvm_irq_routing_entry e;
+ assert(pin < s->gsi_count);
+
e.gsi = irq;
e.type = KVM_IRQ_ROUTING_IRQCHIP;
e.flags = 0;
kvm_add_routing_entry(s, &e);
}
-int kvm_irqchip_commit_routes(KVMState *s)
+void kvm_irqchip_release_virq(KVMState *s, int virq)
{
- s->irq_routes->flags = 0;
- return kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
+ struct kvm_irq_routing_entry *e;
+ int i;
+
+ for (i = 0; i < s->irq_routes->nr; i++) {
+ e = &s->irq_routes->entries[i];
+ if (e->gsi == virq) {
+ s->irq_routes->nr--;
+ *e = s->irq_routes->entries[s->irq_routes->nr];
+ }
+ }
+ clear_gsi(s, virq);
+
+ kvm_irqchip_commit_routes(s);
+}
+
+static unsigned int kvm_hash_msi(uint32_t data)
+{
+ /* This is optimized for IA32 MSI layout. However, no other arch shall
+ * repeat the mistake of not providing a direct MSI injection API. */
+ return data & 0xff;
+}
+
+static void kvm_flush_dynamic_msi_routes(KVMState *s)
+{
+ KVMMSIRoute *route, *next;
+ unsigned int hash;
+
+ for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
+ QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
+ kvm_irqchip_release_virq(s, route->kroute.gsi);
+ QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
+ g_free(route);
+ }
+ }
+}
+
+static int kvm_irqchip_get_virq(KVMState *s)
+{
+ uint32_t *word = s->used_gsi_bitmap;
+ int max_words = ALIGN(s->gsi_count, 32) / 32;
+ int i, bit;
+ bool retry = true;
+
+again:
+ /* Return the lowest unused GSI in the bitmap */
+ for (i = 0; i < max_words; i++) {
+ bit = ffs(~word[i]);
+ if (!bit) {
+ continue;
+ }
+
+ return bit - 1 + i * 32;
+ }
+ if (!s->direct_msi && retry) {
+ retry = false;
+ kvm_flush_dynamic_msi_routes(s);
+ goto again;
+ }
+ return -ENOSPC;
+
+}
+
+static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
+{
+ unsigned int hash = kvm_hash_msi(msg.data);
+ KVMMSIRoute *route;
+
+ QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
+ if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
+ route->kroute.u.msi.address_hi == (msg.address >> 32) &&
+ route->kroute.u.msi.data == msg.data) {
+ return route;
+ }
+ }
+ return NULL;
+}
+
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
+{
+ struct kvm_msi msi;
+ KVMMSIRoute *route;
+
+ if (s->direct_msi) {
+ msi.address_lo = (uint32_t)msg.address;
+ msi.address_hi = msg.address >> 32;
+ msi.data = msg.data;
+ msi.flags = 0;
+ memset(msi.pad, 0, sizeof(msi.pad));
+
+ return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
+ }
+
+ route = kvm_lookup_msi_route(s, msg);
+ if (!route) {
+ int virq;
+
+ virq = kvm_irqchip_get_virq(s);
+ if (virq < 0) {
+ return virq;
+ }
+
+ route = g_malloc(sizeof(KVMMSIRoute));
+ route->kroute.gsi = virq;
+ route->kroute.type = KVM_IRQ_ROUTING_MSI;
+ route->kroute.flags = 0;
+ route->kroute.u.msi.address_lo = (uint32_t)msg.address;
+ route->kroute.u.msi.address_hi = msg.address >> 32;
+ route->kroute.u.msi.data = msg.data;
+
+ kvm_add_routing_entry(s, &route->kroute);
+
+ QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
+ entry);
+ }
+
+ assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
+
+ return kvm_irqchip_set_irq(s, route->kroute.gsi, 1);
+}
+
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+{
+ struct kvm_irq_routing_entry kroute;
+ int virq;
+
+ if (!kvm_irqchip_in_kernel()) {
+ return -ENOSYS;
+ }
+
+ virq = kvm_irqchip_get_virq(s);
+ if (virq < 0) {
+ return virq;
+ }
+
+ kroute.gsi = virq;
+ kroute.type = KVM_IRQ_ROUTING_MSI;
+ kroute.flags = 0;
+ kroute.u.msi.address_lo = (uint32_t)msg.address;
+ kroute.u.msi.address_hi = msg.address >> 32;
+ kroute.u.msi.data = msg.data;
+
+ kvm_add_routing_entry(s, &kroute);
+
+ return virq;
+}
+
+static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
+{
+ struct kvm_irqfd irqfd = {
+ .fd = fd,
+ .gsi = virq,
+ .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
+ };
+
+ if (!kvm_irqchip_in_kernel()) {
+ return -ENOSYS;
+ }
+
+ return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
}
#else /* !KVM_CAP_IRQ_ROUTING */
static void kvm_init_irq_routing(KVMState *s)
{
}
+
+void kvm_irqchip_release_virq(KVMState *s, int virq)
+{
+}
+
+int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
+{
+ abort();
+}
+
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+{
+ abort();
+}
+
+static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
+{
+ abort();
+}
#endif /* !KVM_CAP_IRQ_ROUTING */
+int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq)
+{
+ return kvm_irqchip_assign_irqfd(s, fd, virq, true);
+}
+
+int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq)
+{
+ return kvm_irqchip_assign_irqfd(s, fd, virq, false);
+}
+
static int kvm_irqchip_create(KVMState *s)
{
QemuOptsList *list = qemu_find_opts("machine");
if (QTAILQ_EMPTY(&list->head) ||
!qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
- "kernel_irqchip", false) ||
+ "kernel_irqchip", true) ||
!kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
return 0;
}
s = g_malloc0(sizeof(KVMState));
+ /*
+ * On systems where the kernel can support different base page
+ * sizes, host page size may be different from TARGET_PAGE_SIZE,
+ * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
+ * page size for the system though.
+ */
+ assert(TARGET_PAGE_SIZE <= getpagesize());
+
#ifdef KVM_CAP_SET_GUEST_DEBUG
QTAILQ_INIT(&s->kvm_sw_breakpoints);
#endif
s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
#endif
+#ifdef KVM_CAP_IRQ_ROUTING
+ s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
+#endif
+
ret = kvm_arch_init(s);
if (ret < 0) {
goto err;
}
}
-static int kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
+static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
{
fprintf(stderr, "KVM internal error.");
if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
static void do_kvm_cpu_synchronize_state(void *_env)
{
- CPUState *env = _env;
+ CPUArchState *env = _env;
if (!env->kvm_vcpu_dirty) {
kvm_arch_get_registers(env);
}
}
-void kvm_cpu_synchronize_state(CPUState *env)
+void kvm_cpu_synchronize_state(CPUArchState *env)
{
if (!env->kvm_vcpu_dirty) {
run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
}
}
-void kvm_cpu_synchronize_post_reset(CPUState *env)
+void kvm_cpu_synchronize_post_reset(CPUArchState *env)
{
kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
env->kvm_vcpu_dirty = 0;
}
-void kvm_cpu_synchronize_post_init(CPUState *env)
+void kvm_cpu_synchronize_post_init(CPUArchState *env)
{
kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
env->kvm_vcpu_dirty = 0;
}
-int kvm_cpu_exec(CPUState *env)
+int kvm_cpu_exec(CPUArchState *env)
{
struct kvm_run *run = env->kvm_run;
int ret, run_ret;
return ret;
}
-int kvm_vcpu_ioctl(CPUState *env, int type, ...)
+int kvm_vcpu_ioctl(CPUArchState *env, int type, ...)
{
int ret;
void *arg;
return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
}
+void *kvm_vmalloc(ram_addr_t size)
+{
+#ifdef TARGET_S390X
+ void *mem;
+
+ mem = kvm_arch_vmalloc(size);
+ if (mem) {
+ return mem;
+ }
+#endif
+ return qemu_vmalloc(size);
+}
+
void kvm_setup_guest_memory(void *start, size_t size)
{
if (!kvm_has_sync_mmu()) {
}
#ifdef KVM_CAP_SET_GUEST_DEBUG
-struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
+struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
target_ulong pc)
{
struct kvm_sw_breakpoint *bp;
return NULL;
}
-int kvm_sw_breakpoints_active(CPUState *env)
+int kvm_sw_breakpoints_active(CPUArchState *env)
{
return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
}
struct kvm_set_guest_debug_data {
struct kvm_guest_debug dbg;
- CPUState *env;
+ CPUArchState *env;
int err;
};
static void kvm_invoke_set_guest_debug(void *data)
{
struct kvm_set_guest_debug_data *dbg_data = data;
- CPUState *env = dbg_data->env;
+ CPUArchState *env = dbg_data->env;
dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
}
-int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
+int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
{
struct kvm_set_guest_debug_data data;
return data.err;
}
-int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
+int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
target_ulong len, int type)
{
struct kvm_sw_breakpoint *bp;
- CPUState *env;
+ CPUArchState *env;
int err;
if (type == GDB_BREAKPOINT_SW) {
return 0;
}
-int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
+int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
target_ulong len, int type)
{
struct kvm_sw_breakpoint *bp;
- CPUState *env;
+ CPUArchState *env;
int err;
if (type == GDB_BREAKPOINT_SW) {
return 0;
}
-void kvm_remove_all_breakpoints(CPUState *current_env)
+void kvm_remove_all_breakpoints(CPUArchState *current_env)
{
struct kvm_sw_breakpoint *bp, *next;
KVMState *s = current_env->kvm_state;
- CPUState *env;
+ CPUArchState *env;
QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
#else /* !KVM_CAP_SET_GUEST_DEBUG */
-int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
+int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
{
return -EINVAL;
}
-int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
+int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
target_ulong len, int type)
{
return -EINVAL;
}
-int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
+int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
target_ulong len, int type)
{
return -EINVAL;
}
-void kvm_remove_all_breakpoints(CPUState *current_env)
+void kvm_remove_all_breakpoints(CPUArchState *current_env)
{
}
#endif /* !KVM_CAP_SET_GUEST_DEBUG */
-int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
+int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset)
{
struct kvm_signal_mask *sigmask;
int r;
return r;
}
-int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
+int kvm_set_ioeventfd_mmio(int fd, uint32_t addr, uint32_t val, bool assign,
+ uint32_t size)
{
int ret;
struct kvm_ioeventfd iofd;
iofd.datamatch = val;
iofd.addr = addr;
- iofd.len = 4;
+ iofd.len = size;
iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
iofd.fd = fd;
return 0;
}
-int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
+int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr)
{
return kvm_arch_on_sigbus_vcpu(env, code, addr);
}
projects / qemu.git / blobdiff