[qemu.git] / xen-mapcache.c

/*
 * Copyright (C) 2011       Citrix Ltd.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "config.h"

#include <sys/resource.h>

#include "hw/xen_backend.h"
#include "blockdev.h"
#include "bitmap.h"

#include <xen/hvm/params.h>
#include <sys/mman.h>

#include "xen-mapcache.h"
#include "trace.h"


//#define MAPCACHE_DEBUG

#ifdef MAPCACHE_DEBUG
#  define DPRINTF(fmt, ...) do { \
    fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
} while (0)
#else
#  define DPRINTF(fmt, ...) do { } while (0)
#endif

#if defined(__i386__)
#  define MCACHE_BUCKET_SHIFT 16
#  define MCACHE_MAX_SIZE     (1UL<<31) /* 2GB Cap */
#elif defined(__x86_64__)
#  define MCACHE_BUCKET_SHIFT 20
#  define MCACHE_MAX_SIZE     (1UL<<35) /* 32GB Cap */
#endif
#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)

#define mapcache_lock()   ((void)0)
#define mapcache_unlock() ((void)0)

typedef struct MapCacheEntry {
    target_phys_addr_t paddr_index;
    uint8_t *vaddr_base;
    unsigned long *valid_mapping;
    uint8_t lock;
    target_phys_addr_t size;
    struct MapCacheEntry *next;
} MapCacheEntry;

typedef struct MapCacheRev {
    uint8_t *vaddr_req;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    QTAILQ_ENTRY(MapCacheRev) next;
} MapCacheRev;

typedef struct MapCache {
    MapCacheEntry *entry;
    unsigned long nr_buckets;
    QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;

    /* For most cases (>99.9%), the page address is the same. */
    target_phys_addr_t last_address_index;
    uint8_t *last_address_vaddr;
    unsigned long max_mcache_size;
    unsigned int mcache_bucket_shift;
} MapCache;

static MapCache *mapcache;

static inline int test_bits(int nr, int size, const unsigned long *addr)
{
    unsigned long res = find_next_zero_bit(addr, size + nr, nr);
    if (res >= nr + size)
        return 1;
    else
        return 0;
}

void xen_map_cache_init(void)
{
    unsigned long size;
    struct rlimit rlimit_as;

    mapcache = g_malloc0(sizeof (MapCache));

    QTAILQ_INIT(&mapcache->locked_entries);
    mapcache->last_address_index = -1;

    getrlimit(RLIMIT_AS, &rlimit_as);
    if (rlimit_as.rlim_max < MCACHE_MAX_SIZE) {
        rlimit_as.rlim_cur = rlimit_as.rlim_max;
    } else {
        rlimit_as.rlim_cur = MCACHE_MAX_SIZE;
    }

    setrlimit(RLIMIT_AS, &rlimit_as);
    mapcache->max_mcache_size = rlimit_as.rlim_cur;

    mapcache->nr_buckets =
        (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
          (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
         (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));

    size = mapcache->nr_buckets * sizeof (MapCacheEntry);
    size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
    DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
            mapcache->nr_buckets, size);
    mapcache->entry = g_malloc0(size);
}

static void xen_remap_bucket(MapCacheEntry *entry,
                             target_phys_addr_t size,
                             target_phys_addr_t address_index)
{
    uint8_t *vaddr_base;
    xen_pfn_t *pfns;
    int *err;
    unsigned int i;
    target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;

    trace_xen_remap_bucket(address_index);

    pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
    err = g_malloc0(nb_pfn * sizeof (int));

    if (entry->vaddr_base != NULL) {
        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }
    }
    if (entry->valid_mapping != NULL) {
        g_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    for (i = 0; i < nb_pfn; i++) {
        pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
    }

    vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
                                     pfns, err, nb_pfn);
    if (vaddr_base == NULL) {
        perror("xc_map_foreign_bulk");
        exit(-1);
    }

    entry->vaddr_base = vaddr_base;
    entry->paddr_index = address_index;
    entry->size = size;
    entry->valid_mapping = (unsigned long *) g_malloc0(sizeof(unsigned long) *
            BITS_TO_LONGS(size >> XC_PAGE_SHIFT));

    bitmap_zero(entry->valid_mapping, nb_pfn);
    for (i = 0; i < nb_pfn; i++) {
        if (!err[i]) {
            bitmap_set(entry->valid_mapping, i, 1);
        }
    }

    g_free(pfns);
    g_free(err);
}

uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
                       uint8_t lock)
{
    MapCacheEntry *entry, *pentry = NULL;
    target_phys_addr_t address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
    target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
    target_phys_addr_t __size = size;

    trace_xen_map_cache(phys_addr);

    if (address_index == mapcache->last_address_index && !lock && !__size) {
        trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
        return mapcache->last_address_vaddr + address_offset;
    }

    /* size is always a multiple of MCACHE_BUCKET_SIZE */
    if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE;
    if (__size % MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
    if (!__size)
        __size = MCACHE_BUCKET_SIZE;

    entry = &mapcache->entry[address_index % mapcache->nr_buckets];

    while (entry && entry->lock && entry->vaddr_base &&
            (entry->paddr_index != address_index || entry->size != __size ||
             !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                 entry->valid_mapping))) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        entry = g_malloc0(sizeof (MapCacheEntry));
        pentry->next = entry;
        xen_remap_bucket(entry, __size, address_index);
    } else if (!entry->lock) {
        if (!entry->vaddr_base || entry->paddr_index != address_index ||
                entry->size != __size ||
                !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                    entry->valid_mapping)) {
            xen_remap_bucket(entry, __size, address_index);
        }
    }

    if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                entry->valid_mapping)) {
        mapcache->last_address_index = -1;
        trace_xen_map_cache_return(NULL);
        return NULL;
    }

    mapcache->last_address_index = address_index;
    mapcache->last_address_vaddr = entry->vaddr_base;
    if (lock) {
        MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
        entry->lock++;
        reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
        reventry->paddr_index = mapcache->last_address_index;
        reventry->size = entry->size;
        QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
    }

    trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
    return mapcache->last_address_vaddr + address_offset;
}

ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
{
    MapCacheEntry *entry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == ptr) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
                    reventry->vaddr_req);
        }
        abort();
        return 0;
    }

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
        return 0;
    }
    return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
        ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
}

void xen_invalidate_map_cache_entry(uint8_t *buffer)
{
    MapCacheEntry *entry = NULL, *pentry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    if (mapcache->last_address_vaddr == buffer) {
        mapcache->last_address_index = -1;
    }

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == buffer) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        DPRINTF("%s, could not find %p\n", __func__, buffer);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
        }
        return;
    }
    QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
    g_free(reventry);

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
        return;
    }
    entry->lock--;
    if (entry->lock > 0 || pentry == NULL) {
        return;
    }

    pentry->next = entry->next;
    if (munmap(entry->vaddr_base, entry->size) != 0) {
        perror("unmap fails");
        exit(-1);
    }
    g_free(entry->valid_mapping);
    g_free(entry);
}

void xen_invalidate_map_cache(void)
{
    unsigned long i;
    MapCacheRev *reventry;

    /* Flush pending AIO before destroying the mapcache */
    qemu_aio_flush();

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        DPRINTF("There should be no locked mappings at this time, "
                "but "TARGET_FMT_plx" -> %p is present\n",
                reventry->paddr_index, reventry->vaddr_req);
    }

    mapcache_lock();

    for (i = 0; i < mapcache->nr_buckets; i++) {
        MapCacheEntry *entry = &mapcache->entry[i];

        if (entry->vaddr_base == NULL) {
            continue;
        }

        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }

        entry->paddr_index = 0;
        entry->vaddr_base = NULL;
        entry->size = 0;
        g_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    mapcache->last_address_index = -1;
    mapcache->last_address_vaddr = NULL;

    mapcache_unlock();
}
Commit	Line	Data
432d268c JN	1	/*
	2	* Copyright (C) 2011 Citrix Ltd.
	3	*
	4	* This work is licensed under the terms of the GNU GPL, version 2. See
	5	* the COPYING file in the top-level directory.
	6	*
	7	*/
	8
	9	#include "config.h"
	10
	11	#include <sys/resource.h>
	12
	13	#include "hw/xen_backend.h"
	14	#include "blockdev.h"
ea6c5f8f	15	#include "bitmap.h"
432d268c JN	16
	17	#include <xen/hvm/params.h>
	18	#include <sys/mman.h>
	19
	20	#include "xen-mapcache.h"
	21	#include "trace.h"
	22
	23
	24	//#define MAPCACHE_DEBUG
	25
	26	#ifdef MAPCACHE_DEBUG
	27	# define DPRINTF(fmt, ...) do { \
	28	fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
	29	} while (0)
	30	#else
	31	# define DPRINTF(fmt, ...) do { } while (0)
	32	#endif
	33
	34	#if defined(__i386__)
	35	# define MCACHE_BUCKET_SHIFT 16
ea6c5f8f	36	# define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
432d268c JN	37	#elif defined(__x86_64__)
432d268c JN	38	# define MCACHE_BUCKET_SHIFT 20
ea6c5f8f	39	# define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
432d268c JN	40	#endif
	41	#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
	42
e41d7c69 JK	43	#define mapcache_lock() ((void)0)
	44	#define mapcache_unlock() ((void)0)
	45
432d268c JN	46	typedef struct MapCacheEntry {
	47	target_phys_addr_t paddr_index;
	48	uint8_t *vaddr_base;
c13390cd	49	unsigned long *valid_mapping;
432d268c	50	uint8_t lock;
c13390cd	51	target_phys_addr_t size;
432d268c JN	52	struct MapCacheEntry *next;
	53	} MapCacheEntry;
	54
	55	typedef struct MapCacheRev {
	56	uint8_t *vaddr_req;
	57	target_phys_addr_t paddr_index;
c13390cd	58	target_phys_addr_t size;
432d268c JN	59	QTAILQ_ENTRY(MapCacheRev) next;
	60	} MapCacheRev;
	61
	62	typedef struct MapCache {
	63	MapCacheEntry *entry;
	64	unsigned long nr_buckets;
	65	QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
	66
	67	/* For most cases (>99.9%), the page address is the same. */
	68	target_phys_addr_t last_address_index;
	69	uint8_t *last_address_vaddr;
	70	unsigned long max_mcache_size;
	71	unsigned int mcache_bucket_shift;
	72	} MapCache;
	73
	74	static MapCache *mapcache;
	75
c13390cd SS	76	static inline int test_bits(int nr, int size, const unsigned long *addr)
	77	{
	78	unsigned long res = find_next_zero_bit(addr, size + nr, nr);
	79	if (res >= nr + size)
	80	return 1;
	81	else
	82	return 0;
	83	}
	84
e41d7c69	85	void xen_map_cache_init(void)
432d268c JN	86	{
	87	unsigned long size;
	88	struct rlimit rlimit_as;
	89
7267c094	90	mapcache = g_malloc0(sizeof (MapCache));
432d268c JN	91
	92	QTAILQ_INIT(&mapcache->locked_entries);
	93	mapcache->last_address_index = -1;
	94
	95	getrlimit(RLIMIT_AS, &rlimit_as);
ea6c5f8f JB	96	if (rlimit_as.rlim_max < MCACHE_MAX_SIZE) {
	97	rlimit_as.rlim_cur = rlimit_as.rlim_max;
	98	} else {
	99	rlimit_as.rlim_cur = MCACHE_MAX_SIZE;
	100	}
	101
432d268c	102	setrlimit(RLIMIT_AS, &rlimit_as);
ea6c5f8f	103	mapcache->max_mcache_size = rlimit_as.rlim_cur;
432d268c JN	104
	105	mapcache->nr_buckets =
	106	(((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
	107	(1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
	108	(MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
	109
	110	size = mapcache->nr_buckets * sizeof (MapCacheEntry);
	111	size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
e41d7c69 JK	112	DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
e41d7c69 JK	113	mapcache->nr_buckets, size);
7267c094	114	mapcache->entry = g_malloc0(size);
432d268c JN	115	}
432d268c JN	116
e41d7c69 JK	117	static void xen_remap_bucket(MapCacheEntry *entry,
	118	target_phys_addr_t size,
	119	target_phys_addr_t address_index)
432d268c JN	120	{
	121	uint8_t *vaddr_base;
	122	xen_pfn_t *pfns;
	123	int *err;
ea6c5f8f	124	unsigned int i;
432d268c JN	125	target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
432d268c JN	126
e41d7c69	127	trace_xen_remap_bucket(address_index);
432d268c	128
7267c094 AL	129	pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
7267c094 AL	130	err = g_malloc0(nb_pfn * sizeof (int));
432d268c JN	131
432d268c JN	132	if (entry->vaddr_base != NULL) {
c13390cd	133	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	134	perror("unmap fails");
	135	exit(-1);
	136	}
	137	}
c13390cd	138	if (entry->valid_mapping != NULL) {
7267c094	139	g_free(entry->valid_mapping);
c13390cd SS	140	entry->valid_mapping = NULL;
c13390cd SS	141	}
432d268c JN	142
	143	for (i = 0; i < nb_pfn; i++) {
	144	pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
	145	}
	146
	147	vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ\|PROT_WRITE,
	148	pfns, err, nb_pfn);
	149	if (vaddr_base == NULL) {
	150	perror("xc_map_foreign_bulk");
	151	exit(-1);
	152	}
	153
	154	entry->vaddr_base = vaddr_base;
	155	entry->paddr_index = address_index;
c13390cd	156	entry->size = size;
7267c094	157	entry->valid_mapping = (unsigned long ) g_malloc0(sizeof(unsigned long)
c13390cd	158	BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
432d268c	159
ea6c5f8f JB	160	bitmap_zero(entry->valid_mapping, nb_pfn);
	161	for (i = 0; i < nb_pfn; i++) {
	162	if (!err[i]) {
	163	bitmap_set(entry->valid_mapping, i, 1);
432d268c	164	}
432d268c JN	165	}
432d268c JN	166
7267c094 AL	167	g_free(pfns);
7267c094 AL	168	g_free(err);
432d268c JN	169	}
432d268c JN	170
e41d7c69 JK	171	uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
e41d7c69 JK	172	uint8_t lock)
432d268c JN	173	{
	174	MapCacheEntry entry, pentry = NULL;
	175	target_phys_addr_t address_index = phys_addr >> MCACHE_BUCKET_SHIFT;
	176	target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
c13390cd	177	target_phys_addr_t __size = size;
432d268c	178
e41d7c69	179	trace_xen_map_cache(phys_addr);
432d268c	180
c13390cd	181	if (address_index == mapcache->last_address_index && !lock && !__size) {
e41d7c69	182	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	183	return mapcache->last_address_vaddr + address_offset;
	184	}
	185
c13390cd SS	186	/* size is always a multiple of MCACHE_BUCKET_SIZE */
	187	if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
	188	__size += MCACHE_BUCKET_SIZE;
	189	if (__size % MCACHE_BUCKET_SIZE)
	190	__size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
	191	if (!__size)
	192	__size = MCACHE_BUCKET_SIZE;
	193
432d268c JN	194	entry = &mapcache->entry[address_index % mapcache->nr_buckets];
432d268c JN	195
c13390cd SS	196	while (entry && entry->lock && entry->vaddr_base &&
	197	(entry->paddr_index != address_index \|\| entry->size != __size \|\|
	198	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	199	entry->valid_mapping))) {
432d268c JN	200	pentry = entry;
	201	entry = entry->next;
	202	}
	203	if (!entry) {
7267c094	204	entry = g_malloc0(sizeof (MapCacheEntry));
432d268c	205	pentry->next = entry;
e41d7c69	206	xen_remap_bucket(entry, __size, address_index);
432d268c JN	207	} else if (!entry->lock) {
432d268c JN	208	if (!entry->vaddr_base \|\| entry->paddr_index != address_index \|\|
c13390cd SS	209	entry->size != __size \|\|
	210	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	211	entry->valid_mapping)) {
e41d7c69	212	xen_remap_bucket(entry, __size, address_index);
432d268c JN	213	}
	214	}
	215
c13390cd SS	216	if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
c13390cd SS	217	entry->valid_mapping)) {
432d268c	218	mapcache->last_address_index = -1;
e41d7c69	219	trace_xen_map_cache_return(NULL);
432d268c JN	220	return NULL;
	221	}
	222
	223	mapcache->last_address_index = address_index;
	224	mapcache->last_address_vaddr = entry->vaddr_base;
	225	if (lock) {
7267c094	226	MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
432d268c JN	227	entry->lock++;
	228	reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
	229	reventry->paddr_index = mapcache->last_address_index;
c13390cd	230	reventry->size = entry->size;
432d268c JN	231	QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
	232	}
	233
e41d7c69	234	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	235	return mapcache->last_address_vaddr + address_offset;
	236	}
	237
e41d7c69	238	ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
432d268c	239	{
ecf169b7	240	MapCacheEntry *entry = NULL;
432d268c JN	241	MapCacheRev *reventry;
432d268c JN	242	target_phys_addr_t paddr_index;
c13390cd	243	target_phys_addr_t size;
432d268c JN	244	int found = 0;
	245
	246	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	247	if (reventry->vaddr_req == ptr) {
	248	paddr_index = reventry->paddr_index;
c13390cd	249	size = reventry->size;
432d268c JN	250	found = 1;
	251	break;
	252	}
	253	}
	254	if (!found) {
e41d7c69	255	fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
432d268c JN	256	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	257	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
	258	reventry->vaddr_req);
	259	}
	260	abort();
	261	return 0;
	262	}
	263
c13390cd SS	264	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd SS	265	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
c13390cd SS	266	entry = entry->next;
	267	}
	268	if (!entry) {
	269	DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
	270	return 0;
	271	}
	272	return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
	273	((unsigned long) ptr - (unsigned long) entry->vaddr_base);
432d268c JN	274	}
432d268c JN	275
e41d7c69	276	void xen_invalidate_map_cache_entry(uint8_t *buffer)
432d268c JN	277	{
	278	MapCacheEntry entry = NULL, pentry = NULL;
	279	MapCacheRev *reventry;
	280	target_phys_addr_t paddr_index;
c13390cd	281	target_phys_addr_t size;
432d268c JN	282	int found = 0;
	283
	284	if (mapcache->last_address_vaddr == buffer) {
	285	mapcache->last_address_index = -1;
	286	}
	287
	288	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	289	if (reventry->vaddr_req == buffer) {
	290	paddr_index = reventry->paddr_index;
c13390cd	291	size = reventry->size;
432d268c JN	292	found = 1;
	293	break;
	294	}
	295	}
	296	if (!found) {
e41d7c69	297	DPRINTF("%s, could not find %p\n", __func__, buffer);
432d268c JN	298	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	299	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
	300	}
	301	return;
	302	}
	303	QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
7267c094	304	g_free(reventry);
432d268c JN	305
432d268c JN	306	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd	307	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
432d268c JN	308	pentry = entry;
	309	entry = entry->next;
	310	}
	311	if (!entry) {
	312	DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
	313	return;
	314	}
	315	entry->lock--;
	316	if (entry->lock > 0 \|\| pentry == NULL) {
	317	return;
	318	}
	319
	320	pentry->next = entry->next;
c13390cd	321	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	322	perror("unmap fails");
	323	exit(-1);
	324	}
7267c094 AL	325	g_free(entry->valid_mapping);
7267c094 AL	326	g_free(entry);
432d268c JN	327	}
432d268c JN	328
e41d7c69	329	void xen_invalidate_map_cache(void)
432d268c JN	330	{
	331	unsigned long i;
	332	MapCacheRev *reventry;
	333
	334	/* Flush pending AIO before destroying the mapcache */
	335	qemu_aio_flush();
	336
	337	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	338	DPRINTF("There should be no locked mappings at this time, "
	339	"but "TARGET_FMT_plx" -> %p is present\n",
	340	reventry->paddr_index, reventry->vaddr_req);
	341	}
	342
	343	mapcache_lock();
	344
	345	for (i = 0; i < mapcache->nr_buckets; i++) {
	346	MapCacheEntry *entry = &mapcache->entry[i];
	347
	348	if (entry->vaddr_base == NULL) {
	349	continue;
	350	}
	351
c13390cd	352	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	353	perror("unmap fails");
	354	exit(-1);
	355	}
	356
	357	entry->paddr_index = 0;
	358	entry->vaddr_base = NULL;
c13390cd	359	entry->size = 0;
7267c094	360	g_free(entry->valid_mapping);
c13390cd	361	entry->valid_mapping = NULL;
432d268c JN	362	}
	363
	364	mapcache->last_address_index = -1;
	365	mapcache->last_address_vaddr = NULL;
	366
	367	mapcache_unlock();
	368	}