[linux.git] / drivers / xen / tmem.c

/*
 * Xen implementation for transcendent memory (tmem)
 *
 * Copyright (C) 2009-2011 Oracle Corp.  All rights reserved.
 * Author: Dan Magenheimer
 */

#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/cleancache.h>
#include <linux/frontswap.h>

#include <xen/xen.h>
#include <xen/interface/xen.h>
#include <xen/page.h>
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <xen/tmem.h>

#ifndef CONFIG_XEN_TMEM_MODULE
bool __read_mostly tmem_enabled = false;

static int __init enable_tmem(char *s)
{
	tmem_enabled = true;
	return 1;
}
__setup("tmem", enable_tmem);
#endif

#ifdef CONFIG_CLEANCACHE
static bool cleancache __read_mostly = true;
module_param(cleancache, bool, S_IRUGO);
static bool selfballooning __read_mostly = true;
module_param(selfballooning, bool, S_IRUGO);
#endif /* CONFIG_CLEANCACHE */

#ifdef CONFIG_FRONTSWAP
static bool frontswap __read_mostly = true;
module_param(frontswap, bool, S_IRUGO);
#else /* CONFIG_FRONTSWAP */
#define frontswap (0)
#endif /* CONFIG_FRONTSWAP */

#ifdef CONFIG_XEN_SELFBALLOONING
static bool selfshrinking __read_mostly = true;
module_param(selfshrinking, bool, S_IRUGO);
#endif /* CONFIG_XEN_SELFBALLOONING */

#define TMEM_CONTROL               0
#define TMEM_NEW_POOL              1
#define TMEM_DESTROY_POOL          2
#define TMEM_NEW_PAGE              3
#define TMEM_PUT_PAGE              4
#define TMEM_GET_PAGE              5
#define TMEM_FLUSH_PAGE            6
#define TMEM_FLUSH_OBJECT          7
#define TMEM_READ                  8
#define TMEM_WRITE                 9
#define TMEM_XCHG                 10

/* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
#define TMEM_POOL_PERSIST          1
#define TMEM_POOL_SHARED           2
#define TMEM_POOL_PAGESIZE_SHIFT   4
#define TMEM_VERSION_SHIFT        24


struct tmem_pool_uuid {
	u64 uuid_lo;
	u64 uuid_hi;
};

struct tmem_oid {
	u64 oid[3];
};

#define TMEM_POOL_PRIVATE_UUID	{ 0, 0 }

/* flags for tmem_ops.new_pool */
#define TMEM_POOL_PERSIST          1
#define TMEM_POOL_SHARED           2

/* xen tmem foundation ops/hypercalls */

static inline int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, struct tmem_oid oid,
	u32 index, unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len)
{
	struct tmem_op op;
	int rc = 0;

	op.cmd = tmem_cmd;
	op.pool_id = tmem_pool;
	op.u.gen.oid[0] = oid.oid[0];
	op.u.gen.oid[1] = oid.oid[1];
	op.u.gen.oid[2] = oid.oid[2];
	op.u.gen.index = index;
	op.u.gen.tmem_offset = tmem_offset;
	op.u.gen.pfn_offset = pfn_offset;
	op.u.gen.len = len;
	set_xen_guest_handle(op.u.gen.gmfn, (void *)gmfn);
	rc = HYPERVISOR_tmem_op(&op);
	return rc;
}

static int xen_tmem_new_pool(struct tmem_pool_uuid uuid,
				u32 flags, unsigned long pagesize)
{
	struct tmem_op op;
	int rc = 0, pageshift;

	for (pageshift = 0; pagesize != 1; pageshift++)
		pagesize >>= 1;
	flags |= (pageshift - 12) << TMEM_POOL_PAGESIZE_SHIFT;
	flags |= TMEM_SPEC_VERSION << TMEM_VERSION_SHIFT;
	op.cmd = TMEM_NEW_POOL;
	op.u.new.uuid[0] = uuid.uuid_lo;
	op.u.new.uuid[1] = uuid.uuid_hi;
	op.u.new.flags = flags;
	rc = HYPERVISOR_tmem_op(&op);
	return rc;
}

/* xen generic tmem ops */

static int xen_tmem_put_page(u32 pool_id, struct tmem_oid oid,
			     u32 index, struct page *page)
{
	return xen_tmem_op(TMEM_PUT_PAGE, pool_id, oid, index,
			   xen_page_to_gfn(page), 0, 0, 0);
}

static int xen_tmem_get_page(u32 pool_id, struct tmem_oid oid,
			     u32 index, struct page *page)
{
	return xen_tmem_op(TMEM_GET_PAGE, pool_id, oid, index,
			   xen_page_to_gfn(page), 0, 0, 0);
}

static int xen_tmem_flush_page(u32 pool_id, struct tmem_oid oid, u32 index)
{
	return xen_tmem_op(TMEM_FLUSH_PAGE, pool_id, oid, index,
		0, 0, 0, 0);
}

static int xen_tmem_flush_object(u32 pool_id, struct tmem_oid oid)
{
	return xen_tmem_op(TMEM_FLUSH_OBJECT, pool_id, oid, 0, 0, 0, 0, 0);
}


#ifdef CONFIG_CLEANCACHE
static int xen_tmem_destroy_pool(u32 pool_id)
{
	struct tmem_oid oid = { { 0 } };

	return xen_tmem_op(TMEM_DESTROY_POOL, pool_id, oid, 0, 0, 0, 0, 0);
}

/* cleancache ops */

static void tmem_cleancache_put_page(int pool, struct cleancache_filekey key,
				     pgoff_t index, struct page *page)
{
	u32 ind = (u32) index;
	struct tmem_oid oid = *(struct tmem_oid *)&key;

	if (pool < 0)
		return;
	if (ind != index)
		return;
	mb(); /* ensure page is quiescent; tmem may address it with an alias */
	(void)xen_tmem_put_page((u32)pool, oid, ind, page);
}

static int tmem_cleancache_get_page(int pool, struct cleancache_filekey key,
				    pgoff_t index, struct page *page)
{
	u32 ind = (u32) index;
	struct tmem_oid oid = *(struct tmem_oid *)&key;
	int ret;

	/* translate return values to linux semantics */
	if (pool < 0)
		return -1;
	if (ind != index)
		return -1;
	ret = xen_tmem_get_page((u32)pool, oid, ind, page);
	if (ret == 1)
		return 0;
	else
		return -1;
}

static void tmem_cleancache_flush_page(int pool, struct cleancache_filekey key,
				       pgoff_t index)
{
	u32 ind = (u32) index;
	struct tmem_oid oid = *(struct tmem_oid *)&key;

	if (pool < 0)
		return;
	if (ind != index)
		return;
	(void)xen_tmem_flush_page((u32)pool, oid, ind);
}

static void tmem_cleancache_flush_inode(int pool, struct cleancache_filekey key)
{
	struct tmem_oid oid = *(struct tmem_oid *)&key;

	if (pool < 0)
		return;
	(void)xen_tmem_flush_object((u32)pool, oid);
}

static void tmem_cleancache_flush_fs(int pool)
{
	if (pool < 0)
		return;
	(void)xen_tmem_destroy_pool((u32)pool);
}

static int tmem_cleancache_init_fs(size_t pagesize)
{
	struct tmem_pool_uuid uuid_private = TMEM_POOL_PRIVATE_UUID;

	return xen_tmem_new_pool(uuid_private, 0, pagesize);
}

static int tmem_cleancache_init_shared_fs(uuid_t *uuid, size_t pagesize)
{
	struct tmem_pool_uuid shared_uuid;

	shared_uuid.uuid_lo = *(u64 *)&uuid->b[0];
	shared_uuid.uuid_hi = *(u64 *)&uuid->b[8];
	return xen_tmem_new_pool(shared_uuid, TMEM_POOL_SHARED, pagesize);
}

static const struct cleancache_ops tmem_cleancache_ops = {
	.put_page = tmem_cleancache_put_page,
	.get_page = tmem_cleancache_get_page,
	.invalidate_page = tmem_cleancache_flush_page,
	.invalidate_inode = tmem_cleancache_flush_inode,
	.invalidate_fs = tmem_cleancache_flush_fs,
	.init_shared_fs = tmem_cleancache_init_shared_fs,
	.init_fs = tmem_cleancache_init_fs
};
#endif

#ifdef CONFIG_FRONTSWAP
/* frontswap tmem operations */

/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
static int tmem_frontswap_poolid;

/*
 * Swizzling increases objects per swaptype, increasing tmem concurrency
 * for heavy swaploads.  Later, larger nr_cpus -> larger SWIZ_BITS
 */
#define SWIZ_BITS		4
#define SWIZ_MASK		((1 << SWIZ_BITS) - 1)
#define _oswiz(_type, _ind)	((_type << SWIZ_BITS) | (_ind & SWIZ_MASK))
#define iswiz(_ind)		(_ind >> SWIZ_BITS)

static inline struct tmem_oid oswiz(unsigned type, u32 ind)
{
	struct tmem_oid oid = { .oid = { 0 } };
	oid.oid[0] = _oswiz(type, ind);
	return oid;
}

/* returns 0 if the page was successfully put into frontswap, -1 if not */
static int tmem_frontswap_store(unsigned type, pgoff_t offset,
				   struct page *page)
{
	u64 ind64 = (u64)offset;
	u32 ind = (u32)offset;
	int pool = tmem_frontswap_poolid;
	int ret;

	/* THP isn't supported */
	if (PageTransHuge(page))
		return -1;

	if (pool < 0)
		return -1;
	if (ind64 != ind)
		return -1;
	mb(); /* ensure page is quiescent; tmem may address it with an alias */
	ret = xen_tmem_put_page(pool, oswiz(type, ind), iswiz(ind), page);
	/* translate Xen tmem return values to linux semantics */
	if (ret == 1)
		return 0;
	else
		return -1;
}

/*
 * returns 0 if the page was successfully gotten from frontswap, -1 if
 * was not present (should never happen!)
 */
static int tmem_frontswap_load(unsigned type, pgoff_t offset,
				   struct page *page)
{
	u64 ind64 = (u64)offset;
	u32 ind = (u32)offset;
	int pool = tmem_frontswap_poolid;
	int ret;

	if (pool < 0)
		return -1;
	if (ind64 != ind)
		return -1;
	ret = xen_tmem_get_page(pool, oswiz(type, ind), iswiz(ind), page);
	/* translate Xen tmem return values to linux semantics */
	if (ret == 1)
		return 0;
	else
		return -1;
}

/* flush a single page from frontswap */
static void tmem_frontswap_flush_page(unsigned type, pgoff_t offset)
{
	u64 ind64 = (u64)offset;
	u32 ind = (u32)offset;
	int pool = tmem_frontswap_poolid;

	if (pool < 0)
		return;
	if (ind64 != ind)
		return;
	(void) xen_tmem_flush_page(pool, oswiz(type, ind), iswiz(ind));
}

/* flush all pages from the passed swaptype */
static void tmem_frontswap_flush_area(unsigned type)
{
	int pool = tmem_frontswap_poolid;
	int ind;

	if (pool < 0)
		return;
	for (ind = SWIZ_MASK; ind >= 0; ind--)
		(void)xen_tmem_flush_object(pool, oswiz(type, ind));
}

static void tmem_frontswap_init(unsigned ignored)
{
	struct tmem_pool_uuid private = TMEM_POOL_PRIVATE_UUID;

	/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
	if (tmem_frontswap_poolid < 0)
		tmem_frontswap_poolid =
		    xen_tmem_new_pool(private, TMEM_POOL_PERSIST, PAGE_SIZE);
}

static struct frontswap_ops tmem_frontswap_ops = {
	.store = tmem_frontswap_store,
	.load = tmem_frontswap_load,
	.invalidate_page = tmem_frontswap_flush_page,
	.invalidate_area = tmem_frontswap_flush_area,
	.init = tmem_frontswap_init
};
#endif

static int __init xen_tmem_init(void)
{
	if (!xen_domain())
		return 0;
#ifdef CONFIG_FRONTSWAP
	if (tmem_enabled && frontswap) {
		char *s = "";

		tmem_frontswap_poolid = -1;
		frontswap_register_ops(&tmem_frontswap_ops);
		pr_info("frontswap enabled, RAM provided by Xen Transcendent Memory%s\n",
			s);
	}
#endif
#ifdef CONFIG_CLEANCACHE
	BUILD_BUG_ON(sizeof(struct cleancache_filekey) != sizeof(struct tmem_oid));
	if (tmem_enabled && cleancache) {
		int err;

		err = cleancache_register_ops(&tmem_cleancache_ops);
		if (err)
			pr_warn("xen-tmem: failed to enable cleancache: %d\n",
				err);
		else
			pr_info("cleancache enabled, RAM provided by "
				"Xen Transcendent Memory\n");
	}
#endif
#ifdef CONFIG_XEN_SELFBALLOONING
	/*
	 * There is no point of driving pages to the swap system if they
	 * aren't going anywhere in tmem universe.
	 */
	if (!frontswap) {
		selfshrinking = false;
		selfballooning = false;
	}
	xen_selfballoon_init(selfballooning, selfshrinking);
#endif
	return 0;
}

module_init(xen_tmem_init)
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dan Magenheimer <[email protected]>");
MODULE_DESCRIPTION("Shim to Xen transcendent memory");
Commit	Line	Data
5bc20fc5 DM	1	/*
	2	* Xen implementation for transcendent memory (tmem)
	3	*
afec6e04	4	* Copyright (C) 2009-2011 Oracle Corp. All rights reserved.
5bc20fc5 DM	5	* Author: Dan Magenheimer
	6	*/
	7
283c0972 JP	8	#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
283c0972 JP	9
10a7a077	10	#include <linux/module.h>
5bc20fc5 DM	11	#include <linux/kernel.h>
	12	#include <linux/types.h>
	13	#include <linux/init.h>
	14	#include <linux/pagemap.h>
	15	#include <linux/cleancache.h>
afec6e04	16	#include <linux/frontswap.h>
afec6e04	17
5bc20fc5 DM	18	#include <xen/xen.h>
5bc20fc5 DM	19	#include <xen/interface/xen.h>
a9fd60e2	20	#include <xen/page.h>
5bc20fc5	21	#include <asm/xen/hypercall.h>
5bc20fc5	22	#include <asm/xen/hypervisor.h>
b8b0f559	23	#include <xen/tmem.h>
5bc20fc5	24
0cb401d4 KRW	25	#ifndef CONFIG_XEN_TMEM_MODULE
	26	bool __read_mostly tmem_enabled = false;
	27
	28	static int __init enable_tmem(char *s)
	29	{
	30	tmem_enabled = true;
	31	return 1;
	32	}
	33	__setup("tmem", enable_tmem);
	34	#endif
	35
	36	#ifdef CONFIG_CLEANCACHE
9fd19653	37	static bool cleancache __read_mostly = true;
9fd19653	38	module_param(cleancache, bool, S_IRUGO);
2ca62b04	39	static bool selfballooning __read_mostly = true;
9fd19653	40	module_param(selfballooning, bool, S_IRUGO);
0cb401d4 KRW	41	#endif /* CONFIG_CLEANCACHE */
	42
	43	#ifdef CONFIG_FRONTSWAP
9fd19653	44	static bool frontswap __read_mostly = true;
9fd19653	45	module_param(frontswap, bool, S_IRUGO);
1d7004f0 FC	46	#else /* CONFIG_FRONTSWAP */
1d7004f0 FC	47	#define frontswap (0)
e8f9cb03 KRW	48	#endif /* CONFIG_FRONTSWAP */
e8f9cb03 KRW	49
23972c63	50	#ifdef CONFIG_XEN_SELFBALLOONING
2ca62b04 KRW	51	static bool selfshrinking __read_mostly = true;
2ca62b04 KRW	52	module_param(selfshrinking, bool, S_IRUGO);
23972c63	53	#endif /* CONFIG_XEN_SELFBALLOONING */
0cb401d4	54
5bc20fc5 DM	55	#define TMEM_CONTROL 0
	56	#define TMEM_NEW_POOL 1
	57	#define TMEM_DESTROY_POOL 2
	58	#define TMEM_NEW_PAGE 3
	59	#define TMEM_PUT_PAGE 4
	60	#define TMEM_GET_PAGE 5
	61	#define TMEM_FLUSH_PAGE 6
	62	#define TMEM_FLUSH_OBJECT 7
	63	#define TMEM_READ 8
	64	#define TMEM_WRITE 9
	65	#define TMEM_XCHG 10
	66
	67	/* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
	68	#define TMEM_POOL_PERSIST 1
	69	#define TMEM_POOL_SHARED 2
	70	#define TMEM_POOL_PAGESIZE_SHIFT 4
	71	#define TMEM_VERSION_SHIFT 24
	72
	73
	74	struct tmem_pool_uuid {
	75	u64 uuid_lo;
	76	u64 uuid_hi;
	77	};
	78
	79	struct tmem_oid {
	80	u64 oid[3];
	81	};
	82
	83	#define TMEM_POOL_PRIVATE_UUID { 0, 0 }
	84
	85	/* flags for tmem_ops.new_pool */
	86	#define TMEM_POOL_PERSIST 1
	87	#define TMEM_POOL_SHARED 2
	88
	89	/* xen tmem foundation ops/hypercalls */
	90
	91	static inline int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, struct tmem_oid oid,
	92	u32 index, unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len)
	93	{
	94	struct tmem_op op;
	95	int rc = 0;
	96
	97	op.cmd = tmem_cmd;
	98	op.pool_id = tmem_pool;
	99	op.u.gen.oid[0] = oid.oid[0];
	100	op.u.gen.oid[1] = oid.oid[1];
	101	op.u.gen.oid[2] = oid.oid[2];
	102	op.u.gen.index = index;
	103	op.u.gen.tmem_offset = tmem_offset;
	104	op.u.gen.pfn_offset = pfn_offset;
	105	op.u.gen.len = len;
	106	set_xen_guest_handle(op.u.gen.gmfn, (void *)gmfn);
	107	rc = HYPERVISOR_tmem_op(&op);
	108	return rc;
	109	}
	110
	111	static int xen_tmem_new_pool(struct tmem_pool_uuid uuid,
	112	u32 flags, unsigned long pagesize)
	113	{
	114	struct tmem_op op;
	115	int rc = 0, pageshift;
	116
	117	for (pageshift = 0; pagesize != 1; pageshift++)
	118	pagesize >>= 1;
119	flags \|= (pageshift - 12) << TMEM_POOL_PAGESIZE_SHIFT;
120	flags \|= TMEM_SPEC_VERSION << TMEM_VERSION_SHIFT;
121	op.cmd = TMEM_NEW_POOL;
122	op.u.new.uuid[0] = uuid.uuid_lo;
123	op.u.new.uuid[1] = uuid.uuid_hi;
124	op.u.new.flags = flags;
125	rc = HYPERVISOR_tmem_op(&op);
126	return rc;
127	}
128
129	/* xen generic tmem ops */
130
131	static int xen_tmem_put_page(u32 pool_id, struct tmem_oid oid,
a76e3cc3	132	u32 index, struct page *page)
5bc20fc5	133	{
5bc20fc5	134	return xen_tmem_op(TMEM_PUT_PAGE, pool_id, oid, index,
a76e3cc3	135	xen_page_to_gfn(page), 0, 0, 0);
5bc20fc5 DM	136	}
	137
	138	static int xen_tmem_get_page(u32 pool_id, struct tmem_oid oid,
a76e3cc3	139	u32 index, struct page *page)
5bc20fc5	140	{
5bc20fc5	141	return xen_tmem_op(TMEM_GET_PAGE, pool_id, oid, index,
a76e3cc3	142	xen_page_to_gfn(page), 0, 0, 0);
5bc20fc5 DM	143	}
	144
	145	static int xen_tmem_flush_page(u32 pool_id, struct tmem_oid oid, u32 index)
	146	{
	147	return xen_tmem_op(TMEM_FLUSH_PAGE, pool_id, oid, index,
	148	0, 0, 0, 0);
	149	}
	150
	151	static int xen_tmem_flush_object(u32 pool_id, struct tmem_oid oid)
	152	{
	153	return xen_tmem_op(TMEM_FLUSH_OBJECT, pool_id, oid, 0, 0, 0, 0, 0);
	154	}
	155
5bc20fc5	156
afec6e04 DM	157	#ifdef CONFIG_CLEANCACHE
	158	static int xen_tmem_destroy_pool(u32 pool_id)
	159	{
	160	struct tmem_oid oid = { { 0 } };
	161
	162	return xen_tmem_op(TMEM_DESTROY_POOL, pool_id, oid, 0, 0, 0, 0, 0);
	163	}
	164
5bc20fc5 DM	165	/* cleancache ops */
	166
	167	static void tmem_cleancache_put_page(int pool, struct cleancache_filekey key,
	168	pgoff_t index, struct page *page)
	169	{
	170	u32 ind = (u32) index;
	171	struct tmem_oid oid = (struct tmem_oid )&key;
5bc20fc5 DM	172
	173	if (pool < 0)
	174	return;
	175	if (ind != index)
	176	return;
	177	mb(); /* ensure page is quiescent; tmem may address it with an alias */
a76e3cc3	178	(void)xen_tmem_put_page((u32)pool, oid, ind, page);
5bc20fc5 DM	179	}
	180
	181	static int tmem_cleancache_get_page(int pool, struct cleancache_filekey key,
	182	pgoff_t index, struct page *page)
	183	{
	184	u32 ind = (u32) index;
	185	struct tmem_oid oid = (struct tmem_oid )&key;
5bc20fc5 DM	186	int ret;
	187
	188	/* translate return values to linux semantics */
	189	if (pool < 0)
	190	return -1;
	191	if (ind != index)
	192	return -1;
a76e3cc3	193	ret = xen_tmem_get_page((u32)pool, oid, ind, page);
5bc20fc5 DM	194	if (ret == 1)
	195	return 0;
	196	else
	197	return -1;
	198	}
	199
	200	static void tmem_cleancache_flush_page(int pool, struct cleancache_filekey key,
	201	pgoff_t index)
	202	{
	203	u32 ind = (u32) index;
	204	struct tmem_oid oid = (struct tmem_oid )&key;
	205
	206	if (pool < 0)
	207	return;
	208	if (ind != index)
	209	return;
	210	(void)xen_tmem_flush_page((u32)pool, oid, ind);
	211	}
	212
	213	static void tmem_cleancache_flush_inode(int pool, struct cleancache_filekey key)
	214	{
	215	struct tmem_oid oid = (struct tmem_oid )&key;
	216
	217	if (pool < 0)
	218	return;
	219	(void)xen_tmem_flush_object((u32)pool, oid);
	220	}
	221
	222	static void tmem_cleancache_flush_fs(int pool)
	223	{
	224	if (pool < 0)
	225	return;
	226	(void)xen_tmem_destroy_pool((u32)pool);
	227	}
	228
	229	static int tmem_cleancache_init_fs(size_t pagesize)
	230	{
	231	struct tmem_pool_uuid uuid_private = TMEM_POOL_PRIVATE_UUID;
	232
	233	return xen_tmem_new_pool(uuid_private, 0, pagesize);
	234	}
	235
85787090	236	static int tmem_cleancache_init_shared_fs(uuid_t *uuid, size_t pagesize)
5bc20fc5 DM	237	{
	238	struct tmem_pool_uuid shared_uuid;
	239
85787090 CH	240	shared_uuid.uuid_lo = (u64 )&uuid->b[0];
85787090 CH	241	shared_uuid.uuid_hi = (u64 )&uuid->b[8];
5bc20fc5 DM	242	return xen_tmem_new_pool(shared_uuid, TMEM_POOL_SHARED, pagesize);
	243	}
	244
b3c6de49	245	static const struct cleancache_ops tmem_cleancache_ops = {
5bc20fc5 DM	246	.put_page = tmem_cleancache_put_page,
5bc20fc5 DM	247	.get_page = tmem_cleancache_get_page,
91c6cc9b DM	248	.invalidate_page = tmem_cleancache_flush_page,
	249	.invalidate_inode = tmem_cleancache_flush_inode,
	250	.invalidate_fs = tmem_cleancache_flush_fs,
5bc20fc5 DM	251	.init_shared_fs = tmem_cleancache_init_shared_fs,
	252	.init_fs = tmem_cleancache_init_fs
	253	};
afec6e04	254	#endif
5bc20fc5	255
afec6e04 DM	256	#ifdef CONFIG_FRONTSWAP
	257	/* frontswap tmem operations */
	258
	259	/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
	260	static int tmem_frontswap_poolid;
	261
	262	/*
	263	* Swizzling increases objects per swaptype, increasing tmem concurrency
	264	* for heavy swaploads. Later, larger nr_cpus -> larger SWIZ_BITS
	265	*/
	266	#define SWIZ_BITS 4
	267	#define SWIZ_MASK ((1 << SWIZ_BITS) - 1)
	268	#define _oswiz(_type, _ind) ((_type << SWIZ_BITS) \| (_ind & SWIZ_MASK))
	269	#define iswiz(_ind) (_ind >> SWIZ_BITS)
	270
	271	static inline struct tmem_oid oswiz(unsigned type, u32 ind)
5bc20fc5	272	{
afec6e04 DM	273	struct tmem_oid oid = { .oid = { 0 } };
	274	oid.oid[0] = _oswiz(type, ind);
	275	return oid;
	276	}
5bc20fc5	277
afec6e04	278	/* returns 0 if the page was successfully put into frontswap, -1 if not */
165c8aed	279	static int tmem_frontswap_store(unsigned type, pgoff_t offset,
afec6e04 DM	280	struct page *page)
	281	{
	282	u64 ind64 = (u64)offset;
	283	u32 ind = (u32)offset;
afec6e04 DM	284	int pool = tmem_frontswap_poolid;
	285	int ret;
	286
7ba71669 HY	287	/* THP isn't supported */
	288	if (PageTransHuge(page))
	289	return -1;
	290
afec6e04 DM	291	if (pool < 0)
	292	return -1;
	293	if (ind64 != ind)
	294	return -1;
	295	mb(); /* ensure page is quiescent; tmem may address it with an alias */
a76e3cc3	296	ret = xen_tmem_put_page(pool, oswiz(type, ind), iswiz(ind), page);
afec6e04 DM	297	/* translate Xen tmem return values to linux semantics */
	298	if (ret == 1)
	299	return 0;
	300	else
	301	return -1;
	302	}
	303
	304	/*
	305	* returns 0 if the page was successfully gotten from frontswap, -1 if
	306	* was not present (should never happen!)
	307	*/
165c8aed	308	static int tmem_frontswap_load(unsigned type, pgoff_t offset,
afec6e04 DM	309	struct page *page)
	310	{
	311	u64 ind64 = (u64)offset;
	312	u32 ind = (u32)offset;
afec6e04 DM	313	int pool = tmem_frontswap_poolid;
	314	int ret;
	315
	316	if (pool < 0)
	317	return -1;
	318	if (ind64 != ind)
	319	return -1;
a76e3cc3	320	ret = xen_tmem_get_page(pool, oswiz(type, ind), iswiz(ind), page);
afec6e04 DM	321	/* translate Xen tmem return values to linux semantics */
	322	if (ret == 1)
	323	return 0;
	324	else
	325	return -1;
	326	}
	327
	328	/* flush a single page from frontswap */
	329	static void tmem_frontswap_flush_page(unsigned type, pgoff_t offset)
	330	{
	331	u64 ind64 = (u64)offset;
	332	u32 ind = (u32)offset;
	333	int pool = tmem_frontswap_poolid;
	334
	335	if (pool < 0)
	336	return;
	337	if (ind64 != ind)
	338	return;
	339	(void) xen_tmem_flush_page(pool, oswiz(type, ind), iswiz(ind));
	340	}
	341
	342	/* flush all pages from the passed swaptype */
	343	static void tmem_frontswap_flush_area(unsigned type)
	344	{
	345	int pool = tmem_frontswap_poolid;
	346	int ind;
	347
	348	if (pool < 0)
	349	return;
	350	for (ind = SWIZ_MASK; ind >= 0; ind--)
	351	(void)xen_tmem_flush_object(pool, oswiz(type, ind));
	352	}
	353
	354	static void tmem_frontswap_init(unsigned ignored)
	355	{
	356	struct tmem_pool_uuid private = TMEM_POOL_PRIVATE_UUID;
	357
	358	/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
	359	if (tmem_frontswap_poolid < 0)
	360	tmem_frontswap_poolid =
	361	xen_tmem_new_pool(private, TMEM_POOL_PERSIST, PAGE_SIZE);
	362	}
	363
1e01c968	364	static struct frontswap_ops tmem_frontswap_ops = {
165c8aed KRW	365	.store = tmem_frontswap_store,
165c8aed KRW	366	.load = tmem_frontswap_load,
91c6cc9b DM	367	.invalidate_page = tmem_frontswap_flush_page,
91c6cc9b DM	368	.invalidate_area = tmem_frontswap_flush_area,
afec6e04 DM	369	.init = tmem_frontswap_init
	370	};
	371	#endif
	372
144c61ad	373	static int __init xen_tmem_init(void)
afec6e04	374	{
5bc20fc5 DM	375	if (!xen_domain())
5bc20fc5 DM	376	return 0;
afec6e04	377	#ifdef CONFIG_FRONTSWAP
9fd19653	378	if (tmem_enabled && frontswap) {
afec6e04	379	char *s = "";
afec6e04 DM	380
afec6e04 DM	381	tmem_frontswap_poolid = -1;
d1dc6f1b	382	frontswap_register_ops(&tmem_frontswap_ops);
283c0972 JP	383	pr_info("frontswap enabled, RAM provided by Xen Transcendent Memory%s\n",
283c0972 JP	384	s);
afec6e04 DM	385	}
afec6e04 DM	386	#endif
5bc20fc5	387	#ifdef CONFIG_CLEANCACHE
01b720f3	388	BUILD_BUG_ON(sizeof(struct cleancache_filekey) != sizeof(struct tmem_oid));
9fd19653	389	if (tmem_enabled && cleancache) {
53d85c98 VD	390	int err;
	391
	392	err = cleancache_register_ops(&tmem_cleancache_ops);
	393	if (err)
	394	pr_warn("xen-tmem: failed to enable cleancache: %d\n",
	395	err);
	396	else
	397	pr_info("cleancache enabled, RAM provided by "
	398	"Xen Transcendent Memory\n");
5bc20fc5	399	}
10a7a077 DM	400	#endif
10a7a077 DM	401	#ifdef CONFIG_XEN_SELFBALLOONING
37d46e15 KRW	402	/*
	403	* There is no point of driving pages to the swap system if they
	404	* aren't going anywhere in tmem universe.
	405	*/
	406	if (!frontswap) {
	407	selfshrinking = false;
	408	selfballooning = false;
	409	}
2ca62b04	410	xen_selfballoon_init(selfballooning, selfshrinking);
5bc20fc5 DM	411	#endif
	412	return 0;
	413	}
	414
	415	module_init(xen_tmem_init)
10a7a077 DM	416	MODULE_LICENSE("GPL");
	417	MODULE_AUTHOR("Dan Magenheimer <[email protected]>");
	418	MODULE_DESCRIPTION("Shim to Xen transcendent memory");