[qemu.git] / xen-mapcache.c

/*
 * Copyright (C) 2011       Citrix Ltd.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
 */

#include "config.h"

#include <sys/resource.h>

#include "hw/xen_backend.h"
#include "blockdev.h"
#include "bitmap.h"

#include <xen/hvm/params.h>
#include <sys/mman.h>

#include "xen-mapcache.h"
#include "trace.h"


//#define MAPCACHE_DEBUG

#ifdef MAPCACHE_DEBUG
#  define DPRINTF(fmt, ...) do { \
    fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
} while (0)
#else
#  define DPRINTF(fmt, ...) do { } while (0)
#endif

#if defined(__i386__)
#  define MCACHE_BUCKET_SHIFT 16
#  define MCACHE_MAX_SIZE     (1UL<<31) /* 2GB Cap */
#elif defined(__x86_64__)
#  define MCACHE_BUCKET_SHIFT 20
#  define MCACHE_MAX_SIZE     (1UL<<35) /* 32GB Cap */
#endif
#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)

/* This is the size of the virtual address space reserve to QEMU that will not
 * be use by MapCache.
 * From empirical tests I observed that qemu use 75MB more than the
 * max_mcache_size.
 */
#define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)

#define mapcache_lock()   ((void)0)
#define mapcache_unlock() ((void)0)

typedef struct MapCacheEntry {
    target_phys_addr_t paddr_index;
    uint8_t *vaddr_base;
    unsigned long *valid_mapping;
    uint8_t lock;
    target_phys_addr_t size;
    struct MapCacheEntry *next;
} MapCacheEntry;

typedef struct MapCacheRev {
    uint8_t *vaddr_req;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    QTAILQ_ENTRY(MapCacheRev) next;
} MapCacheRev;

typedef struct MapCache {
    MapCacheEntry *entry;
    unsigned long nr_buckets;
    QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;

    /* For most cases (>99.9%), the page address is the same. */
    target_phys_addr_t last_address_index;
    uint8_t *last_address_vaddr;
    unsigned long max_mcache_size;
    unsigned int mcache_bucket_shift;
} MapCache;

static MapCache *mapcache;

static inline int test_bits(int nr, int size, const unsigned long *addr)
{
    unsigned long res = find_next_zero_bit(addr, size + nr, nr);
    if (res >= nr + size)
        return 1;
    else
        return 0;
}

void xen_map_cache_init(void)
{
    unsigned long size;
    struct rlimit rlimit_as;

    mapcache = g_malloc0(sizeof (MapCache));

    QTAILQ_INIT(&mapcache->locked_entries);
    mapcache->last_address_index = -1;

    if (geteuid() == 0) {
        rlimit_as.rlim_cur = RLIM_INFINITY;
        rlimit_as.rlim_max = RLIM_INFINITY;
        mapcache->max_mcache_size = MCACHE_MAX_SIZE;
    } else {
        getrlimit(RLIMIT_AS, &rlimit_as);
        rlimit_as.rlim_cur = rlimit_as.rlim_max;

        if (rlimit_as.rlim_max != RLIM_INFINITY) {
            fprintf(stderr, "Warning: QEMU's maximum size of virtual"
                    " memory is not infinity.\n");
        }
        if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
            mapcache->max_mcache_size = rlimit_as.rlim_max -
                NON_MCACHE_MEMORY_SIZE;
        } else {
            mapcache->max_mcache_size = MCACHE_MAX_SIZE;
        }
    }

    setrlimit(RLIMIT_AS, &rlimit_as);

    mapcache->nr_buckets =
        (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
          (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
         (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));

    size = mapcache->nr_buckets * sizeof (MapCacheEntry);
    size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
    DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
            mapcache->nr_buckets, size);
    mapcache->entry = g_malloc0(size);
}

static void xen_remap_bucket(MapCacheEntry *entry,
                             target_phys_addr_t size,
                             target_phys_addr_t address_index)
{
    uint8_t *vaddr_base;
    xen_pfn_t *pfns;
    int *err;
    unsigned int i;
    target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;

    trace_xen_remap_bucket(address_index);

    pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
    err = g_malloc0(nb_pfn * sizeof (int));

    if (entry->vaddr_base != NULL) {
        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }
    }
    if (entry->valid_mapping != NULL) {
        g_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    for (i = 0; i < nb_pfn; i++) {
        pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
    }

    vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
                                     pfns, err, nb_pfn);
    if (vaddr_base == NULL) {
        perror("xc_map_foreign_bulk");
        exit(-1);
    }

    entry->vaddr_base = vaddr_base;
    entry->paddr_index = address_index;
    entry->size = size;
    entry->valid_mapping = (unsigned long *) g_malloc0(sizeof(unsigned long) *
            BITS_TO_LONGS(size >> XC_PAGE_SHIFT));

    bitmap_zero(entry->valid_mapping, nb_pfn);
    for (i = 0; i < nb_pfn; i++) {
        if (!err[i]) {
            bitmap_set(entry->valid_mapping, i, 1);
        }
    }

    g_free(pfns);
    g_free(err);
}

uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
                       uint8_t lock)
{
    MapCacheEntry *entry, *pentry = NULL;
    target_phys_addr_t address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
    target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
    target_phys_addr_t __size = size;

    trace_xen_map_cache(phys_addr);

    if (address_index == mapcache->last_address_index && !lock && !__size) {
        trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
        return mapcache->last_address_vaddr + address_offset;
    }

    /* size is always a multiple of MCACHE_BUCKET_SIZE */
    if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE;
    if (__size % MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
    if (!__size)
        __size = MCACHE_BUCKET_SIZE;

    entry = &mapcache->entry[address_index % mapcache->nr_buckets];

    while (entry && entry->lock && entry->vaddr_base &&
            (entry->paddr_index != address_index || entry->size != __size ||
             !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                 entry->valid_mapping))) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        entry = g_malloc0(sizeof (MapCacheEntry));
        pentry->next = entry;
        xen_remap_bucket(entry, __size, address_index);
    } else if (!entry->lock) {
        if (!entry->vaddr_base || entry->paddr_index != address_index ||
                entry->size != __size ||
                !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                    entry->valid_mapping)) {
            xen_remap_bucket(entry, __size, address_index);
        }
    }

    if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                entry->valid_mapping)) {
        mapcache->last_address_index = -1;
        trace_xen_map_cache_return(NULL);
        return NULL;
    }

    mapcache->last_address_index = address_index;
    mapcache->last_address_vaddr = entry->vaddr_base;
    if (lock) {
        MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
        entry->lock++;
        reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
        reventry->paddr_index = mapcache->last_address_index;
        reventry->size = entry->size;
        QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
    }

    trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
    return mapcache->last_address_vaddr + address_offset;
}

ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
{
    MapCacheEntry *entry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == ptr) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
                    reventry->vaddr_req);
        }
        abort();
        return 0;
    }

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
        return 0;
    }
    return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
        ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
}

void xen_invalidate_map_cache_entry(uint8_t *buffer)
{
    MapCacheEntry *entry = NULL, *pentry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    if (mapcache->last_address_vaddr == buffer) {
        mapcache->last_address_index = -1;
    }

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == buffer) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        DPRINTF("%s, could not find %p\n", __func__, buffer);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
        }
        return;
    }
    QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
    g_free(reventry);

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
        return;
    }
    entry->lock--;
    if (entry->lock > 0 || pentry == NULL) {
        return;
    }

    pentry->next = entry->next;
    if (munmap(entry->vaddr_base, entry->size) != 0) {
        perror("unmap fails");
        exit(-1);
    }
    g_free(entry->valid_mapping);
    g_free(entry);
}

void xen_invalidate_map_cache(void)
{
    unsigned long i;
    MapCacheRev *reventry;

    /* Flush pending AIO before destroying the mapcache */
    bdrv_drain_all();

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        DPRINTF("There should be no locked mappings at this time, "
                "but "TARGET_FMT_plx" -> %p is present\n",
                reventry->paddr_index, reventry->vaddr_req);
    }

    mapcache_lock();

    for (i = 0; i < mapcache->nr_buckets; i++) {
        MapCacheEntry *entry = &mapcache->entry[i];

        if (entry->vaddr_base == NULL) {
            continue;
        }

        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }

        entry->paddr_index = 0;
        entry->vaddr_base = NULL;
        entry->size = 0;
        g_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    mapcache->last_address_index = -1;
    mapcache->last_address_vaddr = NULL;

    mapcache_unlock();
}
Commit	Line	Data
432d268c JN	1	/*
	2	* Copyright (C) 2011 Citrix Ltd.
	3	*
	4	* This work is licensed under the terms of the GNU GPL, version 2. See
	5	* the COPYING file in the top-level directory.
	6	*
6b620ca3 PB	7	* Contributions after 2012-01-13 are licensed under the terms of the
6b620ca3 PB	8	* GNU GPL, version 2 or (at your option) any later version.
432d268c JN	9	*/
	10
	11	#include "config.h"
	12
	13	#include <sys/resource.h>
	14
	15	#include "hw/xen_backend.h"
	16	#include "blockdev.h"
ea6c5f8f	17	#include "bitmap.h"
432d268c JN	18
	19	#include <xen/hvm/params.h>
	20	#include <sys/mman.h>
	21
	22	#include "xen-mapcache.h"
	23	#include "trace.h"
	24
	25
	26	//#define MAPCACHE_DEBUG
	27
	28	#ifdef MAPCACHE_DEBUG
	29	# define DPRINTF(fmt, ...) do { \
	30	fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
	31	} while (0)
	32	#else
	33	# define DPRINTF(fmt, ...) do { } while (0)
	34	#endif
	35
	36	#if defined(__i386__)
	37	# define MCACHE_BUCKET_SHIFT 16
ea6c5f8f	38	# define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
432d268c JN	39	#elif defined(__x86_64__)
432d268c JN	40	# define MCACHE_BUCKET_SHIFT 20
ea6c5f8f	41	# define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
432d268c JN	42	#endif
	43	#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
	44
56c119e5 AP	45	/* This is the size of the virtual address space reserve to QEMU that will not
	46	* be use by MapCache.
	47	* From empirical tests I observed that qemu use 75MB more than the
	48	* max_mcache_size.
	49	*/
	50	#define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
	51
e41d7c69 JK	52	#define mapcache_lock() ((void)0)
	53	#define mapcache_unlock() ((void)0)
	54
432d268c JN	55	typedef struct MapCacheEntry {
	56	target_phys_addr_t paddr_index;
	57	uint8_t *vaddr_base;
c13390cd	58	unsigned long *valid_mapping;
432d268c	59	uint8_t lock;
c13390cd	60	target_phys_addr_t size;
432d268c JN	61	struct MapCacheEntry *next;
	62	} MapCacheEntry;
	63
	64	typedef struct MapCacheRev {
	65	uint8_t *vaddr_req;
	66	target_phys_addr_t paddr_index;
c13390cd	67	target_phys_addr_t size;
432d268c JN	68	QTAILQ_ENTRY(MapCacheRev) next;
	69	} MapCacheRev;
	70
	71	typedef struct MapCache {
	72	MapCacheEntry *entry;
	73	unsigned long nr_buckets;
	74	QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
	75
	76	/* For most cases (>99.9%), the page address is the same. */
	77	target_phys_addr_t last_address_index;
	78	uint8_t *last_address_vaddr;
	79	unsigned long max_mcache_size;
	80	unsigned int mcache_bucket_shift;
	81	} MapCache;
	82
	83	static MapCache *mapcache;
	84
c13390cd SS	85	static inline int test_bits(int nr, int size, const unsigned long *addr)
	86	{
	87	unsigned long res = find_next_zero_bit(addr, size + nr, nr);
	88	if (res >= nr + size)
	89	return 1;
	90	else
	91	return 0;
	92	}
	93
e41d7c69	94	void xen_map_cache_init(void)
432d268c JN	95	{
	96	unsigned long size;
	97	struct rlimit rlimit_as;
	98
7267c094	99	mapcache = g_malloc0(sizeof (MapCache));
432d268c JN	100
	101	QTAILQ_INIT(&mapcache->locked_entries);
	102	mapcache->last_address_index = -1;
	103
56c119e5 AP	104	if (geteuid() == 0) {
	105	rlimit_as.rlim_cur = RLIM_INFINITY;
	106	rlimit_as.rlim_max = RLIM_INFINITY;
	107	mapcache->max_mcache_size = MCACHE_MAX_SIZE;
ea6c5f8f	108	} else {
56c119e5 AP	109	getrlimit(RLIMIT_AS, &rlimit_as);
	110	rlimit_as.rlim_cur = rlimit_as.rlim_max;
	111
	112	if (rlimit_as.rlim_max != RLIM_INFINITY) {
	113	fprintf(stderr, "Warning: QEMU's maximum size of virtual"
	114	" memory is not infinity.\n");
	115	}
	116	if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
	117	mapcache->max_mcache_size = rlimit_as.rlim_max -
	118	NON_MCACHE_MEMORY_SIZE;
	119	} else {
	120	mapcache->max_mcache_size = MCACHE_MAX_SIZE;
	121	}
ea6c5f8f JB	122	}
ea6c5f8f JB	123
432d268c	124	setrlimit(RLIMIT_AS, &rlimit_as);
432d268c JN	125
	126	mapcache->nr_buckets =
	127	(((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
	128	(1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
	129	(MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
	130
	131	size = mapcache->nr_buckets * sizeof (MapCacheEntry);
	132	size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
e41d7c69 JK	133	DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
e41d7c69 JK	134	mapcache->nr_buckets, size);
7267c094	135	mapcache->entry = g_malloc0(size);
432d268c JN	136	}
432d268c JN	137
e41d7c69 JK	138	static void xen_remap_bucket(MapCacheEntry *entry,
	139	target_phys_addr_t size,
	140	target_phys_addr_t address_index)
432d268c JN	141	{
	142	uint8_t *vaddr_base;
	143	xen_pfn_t *pfns;
	144	int *err;
ea6c5f8f	145	unsigned int i;
432d268c JN	146	target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
432d268c JN	147
e41d7c69	148	trace_xen_remap_bucket(address_index);
432d268c	149
7267c094 AL	150	pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
7267c094 AL	151	err = g_malloc0(nb_pfn * sizeof (int));
432d268c JN	152
432d268c JN	153	if (entry->vaddr_base != NULL) {
c13390cd	154	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	155	perror("unmap fails");
	156	exit(-1);
	157	}
	158	}
c13390cd	159	if (entry->valid_mapping != NULL) {
7267c094	160	g_free(entry->valid_mapping);
c13390cd SS	161	entry->valid_mapping = NULL;
c13390cd SS	162	}
432d268c JN	163
	164	for (i = 0; i < nb_pfn; i++) {
	165	pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
	166	}
	167
	168	vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ\|PROT_WRITE,
	169	pfns, err, nb_pfn);
	170	if (vaddr_base == NULL) {
	171	perror("xc_map_foreign_bulk");
	172	exit(-1);
	173	}
	174
	175	entry->vaddr_base = vaddr_base;
	176	entry->paddr_index = address_index;
c13390cd	177	entry->size = size;
7267c094	178	entry->valid_mapping = (unsigned long ) g_malloc0(sizeof(unsigned long)
c13390cd	179	BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
432d268c	180
ea6c5f8f JB	181	bitmap_zero(entry->valid_mapping, nb_pfn);
	182	for (i = 0; i < nb_pfn; i++) {
	183	if (!err[i]) {
	184	bitmap_set(entry->valid_mapping, i, 1);
432d268c	185	}
432d268c JN	186	}
432d268c JN	187
7267c094 AL	188	g_free(pfns);
7267c094 AL	189	g_free(err);
432d268c JN	190	}
432d268c JN	191
e41d7c69 JK	192	uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
e41d7c69 JK	193	uint8_t lock)
432d268c JN	194	{
	195	MapCacheEntry entry, pentry = NULL;
	196	target_phys_addr_t address_index = phys_addr >> MCACHE_BUCKET_SHIFT;
	197	target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
c13390cd	198	target_phys_addr_t __size = size;
432d268c	199
e41d7c69	200	trace_xen_map_cache(phys_addr);
432d268c	201
c13390cd	202	if (address_index == mapcache->last_address_index && !lock && !__size) {
e41d7c69	203	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	204	return mapcache->last_address_vaddr + address_offset;
	205	}
	206
c13390cd SS	207	/* size is always a multiple of MCACHE_BUCKET_SIZE */
	208	if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
	209	__size += MCACHE_BUCKET_SIZE;
	210	if (__size % MCACHE_BUCKET_SIZE)
	211	__size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
	212	if (!__size)
	213	__size = MCACHE_BUCKET_SIZE;
	214
432d268c JN	215	entry = &mapcache->entry[address_index % mapcache->nr_buckets];
432d268c JN	216
c13390cd SS	217	while (entry && entry->lock && entry->vaddr_base &&
	218	(entry->paddr_index != address_index \|\| entry->size != __size \|\|
	219	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	220	entry->valid_mapping))) {
432d268c JN	221	pentry = entry;
	222	entry = entry->next;
	223	}
	224	if (!entry) {
7267c094	225	entry = g_malloc0(sizeof (MapCacheEntry));
432d268c	226	pentry->next = entry;
e41d7c69	227	xen_remap_bucket(entry, __size, address_index);
432d268c JN	228	} else if (!entry->lock) {
432d268c JN	229	if (!entry->vaddr_base \|\| entry->paddr_index != address_index \|\|
c13390cd SS	230	entry->size != __size \|\|
	231	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	232	entry->valid_mapping)) {
e41d7c69	233	xen_remap_bucket(entry, __size, address_index);
432d268c JN	234	}
	235	}
	236
c13390cd SS	237	if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
c13390cd SS	238	entry->valid_mapping)) {
432d268c	239	mapcache->last_address_index = -1;
e41d7c69	240	trace_xen_map_cache_return(NULL);
432d268c JN	241	return NULL;
	242	}
	243
	244	mapcache->last_address_index = address_index;
	245	mapcache->last_address_vaddr = entry->vaddr_base;
	246	if (lock) {
7267c094	247	MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
432d268c JN	248	entry->lock++;
	249	reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
	250	reventry->paddr_index = mapcache->last_address_index;
c13390cd	251	reventry->size = entry->size;
432d268c JN	252	QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
	253	}
	254
e41d7c69	255	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	256	return mapcache->last_address_vaddr + address_offset;
	257	}
	258
e41d7c69	259	ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
432d268c	260	{
ecf169b7	261	MapCacheEntry *entry = NULL;
432d268c JN	262	MapCacheRev *reventry;
432d268c JN	263	target_phys_addr_t paddr_index;
c13390cd	264	target_phys_addr_t size;
432d268c JN	265	int found = 0;
	266
	267	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	268	if (reventry->vaddr_req == ptr) {
	269	paddr_index = reventry->paddr_index;
c13390cd	270	size = reventry->size;
432d268c JN	271	found = 1;
	272	break;
	273	}
	274	}
	275	if (!found) {
e41d7c69	276	fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
432d268c JN	277	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	278	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
	279	reventry->vaddr_req);
	280	}
	281	abort();
	282	return 0;
	283	}
	284
c13390cd SS	285	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd SS	286	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
c13390cd SS	287	entry = entry->next;
	288	}
	289	if (!entry) {
	290	DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
	291	return 0;
	292	}
	293	return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
	294	((unsigned long) ptr - (unsigned long) entry->vaddr_base);
432d268c JN	295	}
432d268c JN	296
e41d7c69	297	void xen_invalidate_map_cache_entry(uint8_t *buffer)
432d268c JN	298	{
	299	MapCacheEntry entry = NULL, pentry = NULL;
	300	MapCacheRev *reventry;
	301	target_phys_addr_t paddr_index;
c13390cd	302	target_phys_addr_t size;
432d268c JN	303	int found = 0;
	304
	305	if (mapcache->last_address_vaddr == buffer) {
	306	mapcache->last_address_index = -1;
	307	}
	308
	309	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	310	if (reventry->vaddr_req == buffer) {
	311	paddr_index = reventry->paddr_index;
c13390cd	312	size = reventry->size;
432d268c JN	313	found = 1;
	314	break;
	315	}
	316	}
	317	if (!found) {
e41d7c69	318	DPRINTF("%s, could not find %p\n", __func__, buffer);
432d268c JN	319	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	320	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
	321	}
	322	return;
	323	}
	324	QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
7267c094	325	g_free(reventry);
432d268c JN	326
432d268c JN	327	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd	328	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
432d268c JN	329	pentry = entry;
	330	entry = entry->next;
	331	}
	332	if (!entry) {
	333	DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
	334	return;
	335	}
	336	entry->lock--;
	337	if (entry->lock > 0 \|\| pentry == NULL) {
	338	return;
	339	}
	340
	341	pentry->next = entry->next;
c13390cd	342	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	343	perror("unmap fails");
	344	exit(-1);
	345	}
7267c094 AL	346	g_free(entry->valid_mapping);
7267c094 AL	347	g_free(entry);
432d268c JN	348	}
432d268c JN	349
e41d7c69	350	void xen_invalidate_map_cache(void)
432d268c JN	351	{
	352	unsigned long i;
	353	MapCacheRev *reventry;
	354
	355	/* Flush pending AIO before destroying the mapcache */
922453bc	356	bdrv_drain_all();
432d268c JN	357
	358	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	359	DPRINTF("There should be no locked mappings at this time, "
	360	"but "TARGET_FMT_plx" -> %p is present\n",
	361	reventry->paddr_index, reventry->vaddr_req);
	362	}
	363
	364	mapcache_lock();
	365
	366	for (i = 0; i < mapcache->nr_buckets; i++) {
	367	MapCacheEntry *entry = &mapcache->entry[i];
	368
	369	if (entry->vaddr_base == NULL) {
	370	continue;
	371	}
	372
c13390cd	373	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	374	perror("unmap fails");
	375	exit(-1);
	376	}
	377
	378	entry->paddr_index = 0;
	379	entry->vaddr_base = NULL;
c13390cd	380	entry->size = 0;
7267c094	381	g_free(entry->valid_mapping);
c13390cd	382	entry->valid_mapping = NULL;
432d268c JN	383	}
	384
	385	mapcache->last_address_index = -1;
	386	mapcache->last_address_vaddr = NULL;
	387
	388	mapcache_unlock();
	389	}