[qemu.git] / hw / msix.c

/*
 * MSI-X device support
 *
 * This module includes support for MSI-X in pci devices.
 *
 * Author: Michael S. Tsirkin <[email protected]>
 *
 *  Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin ([email protected])
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
 */

#include "hw.h"
#include "msi.h"
#include "msix.h"
#include "pci.h"
#include "range.h"

#define MSIX_CAP_LENGTH 12

/* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
#define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)

static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
{
    uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
    MSIMessage msg;

    msg.address = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
    msg.data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
    return msg;
}

/*
 * Special API for POWER to configure the vectors through
 * a side channel. Should never be used by devices.
 */
void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)
{
    uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;

    pci_set_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR, msg.address);
    pci_set_long(table_entry + PCI_MSIX_ENTRY_DATA, msg.data);
    table_entry[PCI_MSIX_ENTRY_VECTOR_CTRL] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
}

static uint8_t msix_pending_mask(int vector)
{
    return 1 << (vector % 8);
}

static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)
{
    return dev->msix_pba + vector / 8;
}

static int msix_is_pending(PCIDevice *dev, int vector)
{
    return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
}

static void msix_set_pending(PCIDevice *dev, int vector)
{
    *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
}

static void msix_clr_pending(PCIDevice *dev, int vector)
{
    *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
}

static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
{
    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
    return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
}

static bool msix_is_masked(PCIDevice *dev, int vector)
{
    return msix_vector_masked(dev, vector, dev->msix_function_masked);
}

static void msix_fire_vector_notifier(PCIDevice *dev,
                                      unsigned int vector, bool is_masked)
{
    MSIMessage msg;
    int ret;

    if (!dev->msix_vector_use_notifier) {
        return;
    }
    if (is_masked) {
        dev->msix_vector_release_notifier(dev, vector);
    } else {
        msg = msix_get_message(dev, vector);
        ret = dev->msix_vector_use_notifier(dev, vector, msg);
        assert(ret >= 0);
    }
}

static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
{
    bool is_masked = msix_is_masked(dev, vector);

    if (is_masked == was_masked) {
        return;
    }

    msix_fire_vector_notifier(dev, vector, is_masked);

    if (!is_masked && msix_is_pending(dev, vector)) {
        msix_clr_pending(dev, vector);
        msix_notify(dev, vector);
    }
}

static void msix_update_function_masked(PCIDevice *dev)
{
    dev->msix_function_masked = !msix_enabled(dev) ||
        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
}

/* Handle MSI-X capability config write. */
void msix_write_config(PCIDevice *dev, uint32_t addr,
                       uint32_t val, int len)
{
    unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
    int vector;
    bool was_masked;

    if (!msix_present(dev) || !range_covers_byte(addr, len, enable_pos)) {
        return;
    }

    was_masked = dev->msix_function_masked;
    msix_update_function_masked(dev);

    if (!msix_enabled(dev)) {
        return;
    }

    pci_device_deassert_intx(dev);

    if (dev->msix_function_masked == was_masked) {
        return;
    }

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        msix_handle_mask_update(dev, vector,
                                msix_vector_masked(dev, vector, was_masked));
    }
}

static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
                                     unsigned size)
{
    PCIDevice *dev = opaque;

    return pci_get_long(dev->msix_table + addr);
}

static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
                                  uint64_t val, unsigned size)
{
    PCIDevice *dev = opaque;
    int vector = addr / PCI_MSIX_ENTRY_SIZE;
    bool was_masked;

    was_masked = msix_is_masked(dev, vector);
    pci_set_long(dev->msix_table + addr, val);
    msix_handle_mask_update(dev, vector, was_masked);
}

static const MemoryRegionOps msix_table_mmio_ops = {
    .read = msix_table_mmio_read,
    .write = msix_table_mmio_write,
    /* TODO: MSIX should be LITTLE_ENDIAN. */
    .endianness = DEVICE_NATIVE_ENDIAN,
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
};

static uint64_t msix_pba_mmio_read(void *opaque, target_phys_addr_t addr,
                                   unsigned size)
{
    PCIDevice *dev = opaque;

    return pci_get_long(dev->msix_pba + addr);
}

static const MemoryRegionOps msix_pba_mmio_ops = {
    .read = msix_pba_mmio_read,
    /* TODO: MSIX should be LITTLE_ENDIAN. */
    .endianness = DEVICE_NATIVE_ENDIAN,
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
};

static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
{
    int vector;

    for (vector = 0; vector < nentries; ++vector) {
        unsigned offset =
            vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
        bool was_masked = msix_is_masked(dev, vector);

        dev->msix_table[offset] |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
        msix_handle_mask_update(dev, vector, was_masked);
    }
}

/* Initialize the MSI-X structures */
int msix_init(struct PCIDevice *dev, unsigned short nentries,
              MemoryRegion *table_bar, uint8_t table_bar_nr,
              unsigned table_offset, MemoryRegion *pba_bar,
              uint8_t pba_bar_nr, unsigned pba_offset, uint8_t cap_pos)
{
    int cap;
    unsigned table_size, pba_size;
    uint8_t *config;

    /* Nothing to do if MSI is not supported by interrupt controller */
    if (!msi_supported) {
        return -ENOTSUP;
    }

    if (nentries < 1 || nentries > PCI_MSIX_FLAGS_QSIZE + 1) {
        return -EINVAL;
    }

    table_size = nentries * PCI_MSIX_ENTRY_SIZE;
    pba_size = QEMU_ALIGN_UP(nentries, 64) / 8;

    /* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
    if ((table_bar_nr == pba_bar_nr &&
         ranges_overlap(table_offset, table_size, pba_offset, pba_size)) ||
        table_offset + table_size > memory_region_size(table_bar) ||
        pba_offset + pba_size > memory_region_size(pba_bar) ||
        (table_offset | pba_offset) & PCI_MSIX_FLAGS_BIRMASK) {
        return -EINVAL;
    }

    cap = pci_add_capability(dev, PCI_CAP_ID_MSIX, cap_pos, MSIX_CAP_LENGTH);
    if (cap < 0) {
        return cap;
    }

    dev->msix_cap = cap;
    dev->cap_present |= QEMU_PCI_CAP_MSIX;
    config = dev->config + cap;

    pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
    dev->msix_entries_nr = nentries;
    dev->msix_function_masked = true;

    pci_set_long(config + PCI_MSIX_TABLE, table_offset | table_bar_nr);
    pci_set_long(config + PCI_MSIX_PBA, pba_offset | pba_bar_nr);

    /* Make flags bit writable. */
    dev->wmask[cap + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
                                             MSIX_MASKALL_MASK;

    dev->msix_table = g_malloc0(table_size);
    dev->msix_pba = g_malloc0(pba_size);
    dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);

    msix_mask_all(dev, nentries);

    memory_region_init_io(&dev->msix_table_mmio, &msix_table_mmio_ops, dev,
                          "msix-table", table_size);
    memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);
    memory_region_init_io(&dev->msix_pba_mmio, &msix_pba_mmio_ops, dev,
                          "msix-pba", pba_size);
    memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);

    return 0;
}

int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
                            uint8_t bar_nr)
{
    int ret;
    char *name;

    /*
     * Migration compatibility dictates that this remains a 4k
     * BAR with the vector table in the lower half and PBA in
     * the upper half.  Do not use these elsewhere!
     */
#define MSIX_EXCLUSIVE_BAR_SIZE 4096
#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
#define MSIX_EXCLUSIVE_CAP_OFFSET 0

    if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
        return -EINVAL;
    }

    name = g_strdup_printf("%s-msix", dev->name);
    memory_region_init(&dev->msix_exclusive_bar, name, MSIX_EXCLUSIVE_BAR_SIZE);
    g_free(name);

    ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
                    MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
                    bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
                    MSIX_EXCLUSIVE_CAP_OFFSET);
    if (ret) {
        memory_region_destroy(&dev->msix_exclusive_bar);
        return ret;
    }

    pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,
                     &dev->msix_exclusive_bar);

    return 0;
}

static void msix_free_irq_entries(PCIDevice *dev)
{
    int vector;

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        dev->msix_entry_used[vector] = 0;
        msix_clr_pending(dev, vector);
    }
}

static void msix_clear_all_vectors(PCIDevice *dev)
{
    int vector;

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        msix_clr_pending(dev, vector);
    }
}

/* Clean up resources for the device. */
void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)
{
    if (!msix_present(dev)) {
        return;
    }
    pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
    dev->msix_cap = 0;
    msix_free_irq_entries(dev);
    dev->msix_entries_nr = 0;
    memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
    memory_region_destroy(&dev->msix_pba_mmio);
    g_free(dev->msix_pba);
    dev->msix_pba = NULL;
    memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
    memory_region_destroy(&dev->msix_table_mmio);
    g_free(dev->msix_table);
    dev->msix_table = NULL;
    g_free(dev->msix_entry_used);
    dev->msix_entry_used = NULL;
    dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
}

void msix_uninit_exclusive_bar(PCIDevice *dev)
{
    if (msix_present(dev)) {
        msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
        memory_region_destroy(&dev->msix_exclusive_bar);
    }
}

void msix_save(PCIDevice *dev, QEMUFile *f)
{
    unsigned n = dev->msix_entries_nr;

    if (!msix_present(dev)) {
        return;
    }

    qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
    qemu_put_buffer(f, dev->msix_pba, (n + 7) / 8);
}

/* Should be called after restoring the config space. */
void msix_load(PCIDevice *dev, QEMUFile *f)
{
    unsigned n = dev->msix_entries_nr;
    unsigned int vector;

    if (!msix_present(dev)) {
        return;
    }

    msix_clear_all_vectors(dev);
    qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
    qemu_get_buffer(f, dev->msix_pba, (n + 7) / 8);
    msix_update_function_masked(dev);

    for (vector = 0; vector < n; vector++) {
        msix_handle_mask_update(dev, vector, true);
    }
}

/* Does device support MSI-X? */
int msix_present(PCIDevice *dev)
{
    return dev->cap_present & QEMU_PCI_CAP_MSIX;
}

/* Is MSI-X enabled? */
int msix_enabled(PCIDevice *dev)
{
    return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
         MSIX_ENABLE_MASK);
}

/* Send an MSI-X message */
void msix_notify(PCIDevice *dev, unsigned vector)
{
    MSIMessage msg;

    if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector])
        return;
    if (msix_is_masked(dev, vector)) {
        msix_set_pending(dev, vector);
        return;
    }

    msg = msix_get_message(dev, vector);

    stl_le_phys(msg.address, msg.data);
}

void msix_reset(PCIDevice *dev)
{
    if (!msix_present(dev)) {
        return;
    }
    msix_clear_all_vectors(dev);
    dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
	    ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
    memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
    memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
    msix_mask_all(dev, dev->msix_entries_nr);
}

/* PCI spec suggests that devices make it possible for software to configure
 * less vectors than supported by the device, but does not specify a standard
 * mechanism for devices to do so.
 *
 * We support this by asking devices to declare vectors software is going to
 * actually use, and checking this on the notification path. Devices that
 * don't want to follow the spec suggestion can declare all vectors as used. */

/* Mark vector as used. */
int msix_vector_use(PCIDevice *dev, unsigned vector)
{
    if (vector >= dev->msix_entries_nr)
        return -EINVAL;
    dev->msix_entry_used[vector]++;
    return 0;
}

/* Mark vector as unused. */
void msix_vector_unuse(PCIDevice *dev, unsigned vector)
{
    if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) {
        return;
    }
    if (--dev->msix_entry_used[vector]) {
        return;
    }
    msix_clr_pending(dev, vector);
}

void msix_unuse_all_vectors(PCIDevice *dev)
{
    if (!msix_present(dev)) {
        return;
    }
    msix_free_irq_entries(dev);
}

unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)
{
    return dev->msix_entries_nr;
}

static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)
{
    MSIMessage msg;

    if (msix_is_masked(dev, vector)) {
        return 0;
    }
    msg = msix_get_message(dev, vector);
    return dev->msix_vector_use_notifier(dev, vector, msg);
}

static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
{
    if (msix_is_masked(dev, vector)) {
        return;
    }
    dev->msix_vector_release_notifier(dev, vector);
}

int msix_set_vector_notifiers(PCIDevice *dev,
                              MSIVectorUseNotifier use_notifier,
                              MSIVectorReleaseNotifier release_notifier)
{
    int vector, ret;

    assert(use_notifier && release_notifier);

    dev->msix_vector_use_notifier = use_notifier;
    dev->msix_vector_release_notifier = release_notifier;

    if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
        (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
        for (vector = 0; vector < dev->msix_entries_nr; vector++) {
            ret = msix_set_notifier_for_vector(dev, vector);
            if (ret < 0) {
                goto undo;
            }
        }
    }
    return 0;

undo:
    while (--vector >= 0) {
        msix_unset_notifier_for_vector(dev, vector);
    }
    dev->msix_vector_use_notifier = NULL;
    dev->msix_vector_release_notifier = NULL;
    return ret;
}

void msix_unset_vector_notifiers(PCIDevice *dev)
{
    int vector;

    assert(dev->msix_vector_use_notifier &&
           dev->msix_vector_release_notifier);

    if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
        (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
        for (vector = 0; vector < dev->msix_entries_nr; vector++) {
            msix_unset_notifier_for_vector(dev, vector);
        }
    }
    dev->msix_vector_use_notifier = NULL;
    dev->msix_vector_release_notifier = NULL;
}
Commit	Line	Data
02eb84d0 MT	1	/*
	2	* MSI-X device support
	3	*
	4	* This module includes support for MSI-X in pci devices.
	5	*
	6	* Author: Michael S. Tsirkin <[email protected]>
	7	*
	8	* Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin ([email protected])
	9	*
	10	* This work is licensed under the terms of the GNU GPL, version 2. See
	11	* the COPYING file in the top-level directory.
6b620ca3 PB	12	*
	13	* Contributions after 2012-01-13 are licensed under the terms of the
	14	* GNU GPL, version 2 or (at your option) any later version.
02eb84d0 MT	15	*/
	16
	17	#include "hw.h"
60ba3cc2	18	#include "msi.h"
02eb84d0 MT	19	#include "msix.h"
02eb84d0 MT	20	#include "pci.h"
bf1b0071	21	#include "range.h"
02eb84d0	22
02eb84d0 MT	23	#define MSIX_CAP_LENGTH 12
02eb84d0 MT	24
2760952b MT	25	/* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
2760952b MT	26	#define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
02eb84d0	27	#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
5b5cb086	28	#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
02eb84d0	29
bc4caf49 JK	30	static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
bc4caf49 JK	31	{
d35e428c	32	uint8_t table_entry = dev->msix_table + vector PCI_MSIX_ENTRY_SIZE;
bc4caf49 JK	33	MSIMessage msg;
	34
	35	msg.address = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
	36	msg.data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
	37	return msg;
	38	}
02eb84d0	39
932d4a42 AK	40	/*
	41	* Special API for POWER to configure the vectors through
	42	* a side channel. Should never be used by devices.
	43	*/
	44	void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)
	45	{
	46	uint8_t table_entry = dev->msix_table + vector PCI_MSIX_ENTRY_SIZE;
	47
	48	pci_set_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR, msg.address);
	49	pci_set_long(table_entry + PCI_MSIX_ENTRY_DATA, msg.data);
	50	table_entry[PCI_MSIX_ENTRY_VECTOR_CTRL] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
	51	}
	52
02eb84d0 MT	53	static uint8_t msix_pending_mask(int vector)
	54	{
	55	return 1 << (vector % 8);
	56	}
	57
	58	static uint8_t msix_pending_byte(PCIDevice dev, int vector)
	59	{
d35e428c	60	return dev->msix_pba + vector / 8;
02eb84d0 MT	61	}
	62
	63	static int msix_is_pending(PCIDevice *dev, int vector)
	64	{
	65	return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
	66	}
	67
	68	static void msix_set_pending(PCIDevice *dev, int vector)
	69	{
	70	*msix_pending_byte(dev, vector) \|= msix_pending_mask(vector);
	71	}
	72
	73	static void msix_clr_pending(PCIDevice *dev, int vector)
	74	{
	75	*msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
	76	}
	77
ae392c41	78	static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
02eb84d0	79	{
ae392c41	80	unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
d35e428c	81	return fmask \|\| dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
5b5cb086 MT	82	}
5b5cb086 MT	83
ae392c41	84	static bool msix_is_masked(PCIDevice *dev, int vector)
5b5cb086	85	{
ae392c41 MT	86	return msix_vector_masked(dev, vector, dev->msix_function_masked);
	87	}
	88
2cdfe53c JK	89	static void msix_fire_vector_notifier(PCIDevice *dev,
	90	unsigned int vector, bool is_masked)
	91	{
	92	MSIMessage msg;
	93	int ret;
	94
	95	if (!dev->msix_vector_use_notifier) {
	96	return;
	97	}
	98	if (is_masked) {
	99	dev->msix_vector_release_notifier(dev, vector);
	100	} else {
	101	msg = msix_get_message(dev, vector);
	102	ret = dev->msix_vector_use_notifier(dev, vector, msg);
	103	assert(ret >= 0);
	104	}
	105	}
	106
ae392c41 MT	107	static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
	108	{
	109	bool is_masked = msix_is_masked(dev, vector);
2cdfe53c	110
ae392c41 MT	111	if (is_masked == was_masked) {
	112	return;
	113	}
	114
2cdfe53c JK	115	msix_fire_vector_notifier(dev, vector, is_masked);
2cdfe53c JK	116
ae392c41	117	if (!is_masked && msix_is_pending(dev, vector)) {
5b5cb086 MT	118	msix_clr_pending(dev, vector);
	119	msix_notify(dev, vector);
	120	}
	121	}
	122
50322249 MT	123	static void msix_update_function_masked(PCIDevice *dev)
	124	{
	125	dev->msix_function_masked = !msix_enabled(dev) \|\|
	126	(dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
	127	}
	128
5b5cb086 MT	129	/* Handle MSI-X capability config write. */
	130	void msix_write_config(PCIDevice *dev, uint32_t addr,
	131	uint32_t val, int len)
	132	{
	133	unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
	134	int vector;
50322249	135	bool was_masked;
5b5cb086	136
7c9958b0	137	if (!msix_present(dev) \|\| !range_covers_byte(addr, len, enable_pos)) {
5b5cb086 MT	138	return;
	139	}
	140
50322249 MT	141	was_masked = dev->msix_function_masked;
	142	msix_update_function_masked(dev);
	143
5b5cb086 MT	144	if (!msix_enabled(dev)) {
	145	return;
	146	}
	147
e407bf13	148	pci_device_deassert_intx(dev);
5b5cb086	149
50322249	150	if (dev->msix_function_masked == was_masked) {
5b5cb086 MT	151	return;
	152	}
	153
	154	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
ae392c41 MT	155	msix_handle_mask_update(dev, vector,
ae392c41 MT	156	msix_vector_masked(dev, vector, was_masked));
5b5cb086	157	}
02eb84d0 MT	158	}
02eb84d0 MT	159
d35e428c AW	160	static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
d35e428c AW	161	unsigned size)
eebcb0a7 AW	162	{
eebcb0a7 AW	163	PCIDevice *dev = opaque;
eebcb0a7	164
d35e428c	165	return pci_get_long(dev->msix_table + addr);
eebcb0a7 AW	166	}
eebcb0a7 AW	167
d35e428c AW	168	static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
d35e428c AW	169	uint64_t val, unsigned size)
02eb84d0 MT	170	{
02eb84d0 MT	171	PCIDevice *dev = opaque;
d35e428c	172	int vector = addr / PCI_MSIX_ENTRY_SIZE;
ae392c41	173	bool was_masked;
9a93b617	174
ae392c41	175	was_masked = msix_is_masked(dev, vector);
d35e428c	176	pci_set_long(dev->msix_table + addr, val);
ae392c41	177	msix_handle_mask_update(dev, vector, was_masked);
02eb84d0 MT	178	}
02eb84d0 MT	179
d35e428c AW	180	static const MemoryRegionOps msix_table_mmio_ops = {
	181	.read = msix_table_mmio_read,
	182	.write = msix_table_mmio_write,
	183	/* TODO: MSIX should be LITTLE_ENDIAN. */
	184	.endianness = DEVICE_NATIVE_ENDIAN,
	185	.valid = {
	186	.min_access_size = 4,
	187	.max_access_size = 4,
	188	},
	189	};
	190
	191	static uint64_t msix_pba_mmio_read(void *opaque, target_phys_addr_t addr,
	192	unsigned size)
	193	{
	194	PCIDevice *dev = opaque;
	195
	196	return pci_get_long(dev->msix_pba + addr);
	197	}
	198
	199	static const MemoryRegionOps msix_pba_mmio_ops = {
	200	.read = msix_pba_mmio_read,
2cf62ad7	201	/* TODO: MSIX should be LITTLE_ENDIAN. */
95524ae8 AK	202	.endianness = DEVICE_NATIVE_ENDIAN,
	203	.valid = {
	204	.min_access_size = 4,
	205	.max_access_size = 4,
	206	},
02eb84d0 MT	207	};
02eb84d0 MT	208
ae1be0bb MT	209	static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
	210	{
	211	int vector;
5b5f1330	212
ae1be0bb	213	for (vector = 0; vector < nentries; ++vector) {
01731cfb JK	214	unsigned offset =
01731cfb JK	215	vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
5b5f1330 JK	216	bool was_masked = msix_is_masked(dev, vector);
5b5f1330 JK	217
d35e428c	218	dev->msix_table[offset] \|= PCI_MSIX_ENTRY_CTRL_MASKBIT;
5b5f1330	219	msix_handle_mask_update(dev, vector, was_masked);
ae1be0bb MT	220	}
	221	}
	222
5a2c2029	223	/* Initialize the MSI-X structures */
02eb84d0	224	int msix_init(struct PCIDevice *dev, unsigned short nentries,
5a2c2029 AW	225	MemoryRegion *table_bar, uint8_t table_bar_nr,
	226	unsigned table_offset, MemoryRegion *pba_bar,
	227	uint8_t pba_bar_nr, unsigned pba_offset, uint8_t cap_pos)
02eb84d0	228	{
5a2c2029	229	int cap;
d35e428c	230	unsigned table_size, pba_size;
5a2c2029	231	uint8_t *config;
60ba3cc2	232
02eb84d0	233	/* Nothing to do if MSI is not supported by interrupt controller */
60ba3cc2	234	if (!msi_supported) {
02eb84d0	235	return -ENOTSUP;
60ba3cc2	236	}
5a2c2029 AW	237
5a2c2029 AW	238	if (nentries < 1 \|\| nentries > PCI_MSIX_FLAGS_QSIZE + 1) {
02eb84d0	239	return -EINVAL;
5a2c2029	240	}
02eb84d0	241
d35e428c AW	242	table_size = nentries * PCI_MSIX_ENTRY_SIZE;
	243	pba_size = QEMU_ALIGN_UP(nentries, 64) / 8;
	244
5a2c2029 AW	245	/* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
	246	if ((table_bar_nr == pba_bar_nr &&
	247	ranges_overlap(table_offset, table_size, pba_offset, pba_size)) \|\|
	248	table_offset + table_size > memory_region_size(table_bar) \|\|
	249	pba_offset + pba_size > memory_region_size(pba_bar) \|\|
	250	(table_offset \| pba_offset) & PCI_MSIX_FLAGS_BIRMASK) {
	251	return -EINVAL;
	252	}
	253
	254	cap = pci_add_capability(dev, PCI_CAP_ID_MSIX, cap_pos, MSIX_CAP_LENGTH);
	255	if (cap < 0) {
	256	return cap;
	257	}
	258
	259	dev->msix_cap = cap;
	260	dev->cap_present \|= QEMU_PCI_CAP_MSIX;
	261	config = dev->config + cap;
	262
	263	pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
	264	dev->msix_entries_nr = nentries;
	265	dev->msix_function_masked = true;
	266
	267	pci_set_long(config + PCI_MSIX_TABLE, table_offset \| table_bar_nr);
	268	pci_set_long(config + PCI_MSIX_PBA, pba_offset \| pba_bar_nr);
	269
	270	/* Make flags bit writable. */
	271	dev->wmask[cap + MSIX_CONTROL_OFFSET] \|= MSIX_ENABLE_MASK \|
	272	MSIX_MASKALL_MASK;
02eb84d0	273
d35e428c AW	274	dev->msix_table = g_malloc0(table_size);
d35e428c AW	275	dev->msix_pba = g_malloc0(pba_size);
5a2c2029 AW	276	dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);
5a2c2029 AW	277
ae1be0bb	278	msix_mask_all(dev, nentries);
02eb84d0	279
d35e428c AW	280	memory_region_init_io(&dev->msix_table_mmio, &msix_table_mmio_ops, dev,
d35e428c AW	281	"msix-table", table_size);
5a2c2029	282	memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);
d35e428c AW	283	memory_region_init_io(&dev->msix_pba_mmio, &msix_pba_mmio_ops, dev,
d35e428c AW	284	"msix-pba", pba_size);
5a2c2029	285	memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);
02eb84d0	286
02eb84d0	287	return 0;
02eb84d0 MT	288	}
02eb84d0 MT	289
53f94925 AW	290	int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
	291	uint8_t bar_nr)
	292	{
	293	int ret;
	294	char *name;
	295
	296	/*
	297	* Migration compatibility dictates that this remains a 4k
	298	* BAR with the vector table in the lower half and PBA in
	299	* the upper half. Do not use these elsewhere!
	300	*/
	301	#define MSIX_EXCLUSIVE_BAR_SIZE 4096
5a2c2029	302	#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
53f94925	303	#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
5a2c2029	304	#define MSIX_EXCLUSIVE_CAP_OFFSET 0
53f94925 AW	305
	306	if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
	307	return -EINVAL;
	308	}
	309
5f893b4e	310	name = g_strdup_printf("%s-msix", dev->name);
53f94925	311	memory_region_init(&dev->msix_exclusive_bar, name, MSIX_EXCLUSIVE_BAR_SIZE);
5f893b4e	312	g_free(name);
53f94925 AW	313
53f94925 AW	314	ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
5a2c2029 AW	315	MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
	316	bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
	317	MSIX_EXCLUSIVE_CAP_OFFSET);
53f94925 AW	318	if (ret) {
	319	memory_region_destroy(&dev->msix_exclusive_bar);
	320	return ret;
	321	}
	322
	323	pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,
	324	&dev->msix_exclusive_bar);
	325
	326	return 0;
	327	}
	328
98304c84 MT	329	static void msix_free_irq_entries(PCIDevice *dev)
	330	{
	331	int vector;
	332
	333	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
	334	dev->msix_entry_used[vector] = 0;
	335	msix_clr_pending(dev, vector);
	336	}
	337	}
	338
3cac001e MT	339	static void msix_clear_all_vectors(PCIDevice *dev)
	340	{
	341	int vector;
	342
	343	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
	344	msix_clr_pending(dev, vector);
	345	}
	346	}
	347
02eb84d0	348	/* Clean up resources for the device. */
572992ee	349	void msix_uninit(PCIDevice dev, MemoryRegion table_bar, MemoryRegion *pba_bar)
02eb84d0	350	{
44701ab7	351	if (!msix_present(dev)) {
572992ee	352	return;
44701ab7	353	}
02eb84d0 MT	354	pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
	355	dev->msix_cap = 0;
	356	msix_free_irq_entries(dev);
	357	dev->msix_entries_nr = 0;
5a2c2029	358	memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
d35e428c AW	359	memory_region_destroy(&dev->msix_pba_mmio);
	360	g_free(dev->msix_pba);
	361	dev->msix_pba = NULL;
5a2c2029	362	memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
d35e428c AW	363	memory_region_destroy(&dev->msix_table_mmio);
	364	g_free(dev->msix_table);
	365	dev->msix_table = NULL;
7267c094	366	g_free(dev->msix_entry_used);
02eb84d0 MT	367	dev->msix_entry_used = NULL;
02eb84d0 MT	368	dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
02eb84d0 MT	369	}
02eb84d0 MT	370
53f94925 AW	371	void msix_uninit_exclusive_bar(PCIDevice *dev)
	372	{
	373	if (msix_present(dev)) {
5a2c2029	374	msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
53f94925 AW	375	memory_region_destroy(&dev->msix_exclusive_bar);
	376	}
	377	}
	378
02eb84d0 MT	379	void msix_save(PCIDevice dev, QEMUFile f)
02eb84d0 MT	380	{
9a3e12c8 MT	381	unsigned n = dev->msix_entries_nr;
9a3e12c8 MT	382
44701ab7	383	if (!msix_present(dev)) {
9a3e12c8	384	return;
72755a70	385	}
9a3e12c8	386
d35e428c AW	387	qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	388	qemu_put_buffer(f, dev->msix_pba, (n + 7) / 8);
02eb84d0 MT	389	}
	390
	391	/* Should be called after restoring the config space. */
	392	void msix_load(PCIDevice dev, QEMUFile f)
	393	{
	394	unsigned n = dev->msix_entries_nr;
2cdfe53c	395	unsigned int vector;
02eb84d0	396
44701ab7	397	if (!msix_present(dev)) {
02eb84d0	398	return;
98846d73	399	}
02eb84d0	400
3cac001e	401	msix_clear_all_vectors(dev);
d35e428c AW	402	qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	403	qemu_get_buffer(f, dev->msix_pba, (n + 7) / 8);
50322249	404	msix_update_function_masked(dev);
2cdfe53c JK	405
	406	for (vector = 0; vector < n; vector++) {
	407	msix_handle_mask_update(dev, vector, true);
	408	}
02eb84d0 MT	409	}
	410
	411	/* Does device support MSI-X? */
	412	int msix_present(PCIDevice *dev)
	413	{
	414	return dev->cap_present & QEMU_PCI_CAP_MSIX;
	415	}
	416
	417	/* Is MSI-X enabled? */
	418	int msix_enabled(PCIDevice *dev)
	419	{
	420	return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
2760952b	421	(dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
02eb84d0 MT	422	MSIX_ENABLE_MASK);
	423	}
	424
02eb84d0 MT	425	/* Send an MSI-X message */
	426	void msix_notify(PCIDevice *dev, unsigned vector)
	427	{
bc4caf49	428	MSIMessage msg;
02eb84d0 MT	429
	430	if (vector >= dev->msix_entries_nr \|\| !dev->msix_entry_used[vector])
	431	return;
	432	if (msix_is_masked(dev, vector)) {
	433	msix_set_pending(dev, vector);
	434	return;
	435	}
	436
bc4caf49 JK	437	msg = msix_get_message(dev, vector);
	438
	439	stl_le_phys(msg.address, msg.data);
02eb84d0 MT	440	}
	441
	442	void msix_reset(PCIDevice *dev)
	443	{
44701ab7	444	if (!msix_present(dev)) {
02eb84d0	445	return;
44701ab7	446	}
3cac001e	447	msix_clear_all_vectors(dev);
2760952b MT	448	dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
2760952b MT	449	~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
d35e428c AW	450	memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	451	memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
ae1be0bb	452	msix_mask_all(dev, dev->msix_entries_nr);
02eb84d0 MT	453	}
	454
	455	/* PCI spec suggests that devices make it possible for software to configure
	456	* less vectors than supported by the device, but does not specify a standard
	457	* mechanism for devices to do so.
	458	*
	459	* We support this by asking devices to declare vectors software is going to
	460	* actually use, and checking this on the notification path. Devices that
	461	* don't want to follow the spec suggestion can declare all vectors as used. */
	462
	463	/* Mark vector as used. */
	464	int msix_vector_use(PCIDevice *dev, unsigned vector)
	465	{
	466	if (vector >= dev->msix_entries_nr)
	467	return -EINVAL;
	468	dev->msix_entry_used[vector]++;
	469	return 0;
	470	}
	471
	472	/* Mark vector as unused. */
	473	void msix_vector_unuse(PCIDevice *dev, unsigned vector)
	474	{
98304c84 MT	475	if (vector >= dev->msix_entries_nr \|\| !dev->msix_entry_used[vector]) {
	476	return;
	477	}
	478	if (--dev->msix_entry_used[vector]) {
	479	return;
	480	}
	481	msix_clr_pending(dev, vector);
02eb84d0	482	}
b5f28bca MT	483
	484	void msix_unuse_all_vectors(PCIDevice *dev)
	485	{
44701ab7	486	if (!msix_present(dev)) {
b5f28bca	487	return;
44701ab7	488	}
b5f28bca MT	489	msix_free_irq_entries(dev);
b5f28bca MT	490	}
2cdfe53c	491
cb697aaa JK	492	unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)
	493	{
	494	return dev->msix_entries_nr;
	495	}
	496
2cdfe53c JK	497	static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)
	498	{
	499	MSIMessage msg;
	500
	501	if (msix_is_masked(dev, vector)) {
	502	return 0;
	503	}
	504	msg = msix_get_message(dev, vector);
	505	return dev->msix_vector_use_notifier(dev, vector, msg);
	506	}
	507
	508	static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
	509	{
	510	if (msix_is_masked(dev, vector)) {
	511	return;
	512	}
	513	dev->msix_vector_release_notifier(dev, vector);
	514	}
	515
	516	int msix_set_vector_notifiers(PCIDevice *dev,
	517	MSIVectorUseNotifier use_notifier,
	518	MSIVectorReleaseNotifier release_notifier)
	519	{
	520	int vector, ret;
	521
	522	assert(use_notifier && release_notifier);
	523
	524	dev->msix_vector_use_notifier = use_notifier;
	525	dev->msix_vector_release_notifier = release_notifier;
	526
	527	if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
	528	(MSIX_ENABLE_MASK \| MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
	529	for (vector = 0; vector < dev->msix_entries_nr; vector++) {
	530	ret = msix_set_notifier_for_vector(dev, vector);
	531	if (ret < 0) {
	532	goto undo;
	533	}
	534	}
	535	}
	536	return 0;
	537
	538	undo:
	539	while (--vector >= 0) {
	540	msix_unset_notifier_for_vector(dev, vector);
	541	}
	542	dev->msix_vector_use_notifier = NULL;
	543	dev->msix_vector_release_notifier = NULL;
	544	return ret;
	545	}
	546
	547	void msix_unset_vector_notifiers(PCIDevice *dev)
	548	{
	549	int vector;
	550
	551	assert(dev->msix_vector_use_notifier &&
	552	dev->msix_vector_release_notifier);
	553
	554	if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
	555	(MSIX_ENABLE_MASK \| MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
	556	for (vector = 0; vector < dev->msix_entries_nr; vector++) {
	557	msix_unset_notifier_for_vector(dev, vector);
	558	}
	559	}
	560	dev->msix_vector_use_notifier = NULL;
561	dev->msix_vector_release_notifier = NULL;
562	}