[linux.git] / mm / slab.h

#ifndef MM_SLAB_H
#define MM_SLAB_H
/*
 * Internal slab definitions
 */

/*
 * State of the slab allocator.
 *
 * This is used to describe the states of the allocator during bootup.
 * Allocators use this to gradually bootstrap themselves. Most allocators
 * have the problem that the structures used for managing slab caches are
 * allocated from slab caches themselves.
 */
enum slab_state {
	DOWN,			/* No slab functionality yet */
	PARTIAL,		/* SLUB: kmem_cache_node available */
	PARTIAL_ARRAYCACHE,	/* SLAB: kmalloc size for arraycache available */
	PARTIAL_NODE,		/* SLAB: kmalloc size for node struct available */
	UP,			/* Slab caches usable but not all extras yet */
	FULL			/* Everything is working */
};

extern enum slab_state slab_state;

/* The slab cache mutex protects the management structures during changes */
extern struct mutex slab_mutex;

/* The list of all slab caches on the system */
extern struct list_head slab_caches;

/* The slab cache that manages slab cache information */
extern struct kmem_cache *kmem_cache;

unsigned long calculate_alignment(unsigned long flags,
		unsigned long align, unsigned long size);

#ifndef CONFIG_SLOB
/* Kmalloc array related functions */
void create_kmalloc_caches(unsigned long);

/* Find the kmalloc slab corresponding for a certain size */
struct kmem_cache *kmalloc_slab(size_t, gfp_t);
#endif


/* Functions provided by the slab allocators */
extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);

extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
			unsigned long flags);
extern void create_boot_cache(struct kmem_cache *, const char *name,
			size_t size, unsigned long flags);

struct mem_cgroup;
#ifdef CONFIG_SLUB
struct kmem_cache *
__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
		   size_t align, unsigned long flags, void (*ctor)(void *));
#else
static inline struct kmem_cache *
__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
		   size_t align, unsigned long flags, void (*ctor)(void *))
{ return NULL; }
#endif


/* Legal flag mask for kmem_cache_create(), for various configurations */
#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
			 SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )

#if defined(CONFIG_DEBUG_SLAB)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
#elif defined(CONFIG_SLUB_DEBUG)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
			  SLAB_TRACE | SLAB_DEBUG_FREE)
#else
#define SLAB_DEBUG_FLAGS (0)
#endif

#if defined(CONFIG_SLAB)
#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
			  SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK)
#elif defined(CONFIG_SLUB)
#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
			  SLAB_TEMPORARY | SLAB_NOTRACK)
#else
#define SLAB_CACHE_FLAGS (0)
#endif

#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)

int __kmem_cache_shutdown(struct kmem_cache *);

struct seq_file;
struct file;

struct slabinfo {
	unsigned long active_objs;
	unsigned long num_objs;
	unsigned long active_slabs;
	unsigned long num_slabs;
	unsigned long shared_avail;
	unsigned int limit;
	unsigned int batchcount;
	unsigned int shared;
	unsigned int objects_per_slab;
	unsigned int cache_order;
};

void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos);

#ifdef CONFIG_MEMCG_KMEM
static inline bool is_root_cache(struct kmem_cache *s)
{
	return !s->memcg_params || s->memcg_params->is_root_cache;
}

static inline bool cache_match_memcg(struct kmem_cache *cachep,
				     struct mem_cgroup *memcg)
{
	return (is_root_cache(cachep) && !memcg) ||
				(cachep->memcg_params->memcg == memcg);
}

static inline void memcg_bind_pages(struct kmem_cache *s, int order)
{
	if (!is_root_cache(s))
		atomic_add(1 << order, &s->memcg_params->nr_pages);
}

static inline void memcg_release_pages(struct kmem_cache *s, int order)
{
	if (is_root_cache(s))
		return;

	if (atomic_sub_and_test((1 << order), &s->memcg_params->nr_pages))
		mem_cgroup_destroy_cache(s);
}

static inline bool slab_equal_or_root(struct kmem_cache *s,
					struct kmem_cache *p)
{
	return (p == s) ||
		(s->memcg_params && (p == s->memcg_params->root_cache));
}

/*
 * We use suffixes to the name in memcg because we can't have caches
 * created in the system with the same name. But when we print them
 * locally, better refer to them with the base name
 */
static inline const char *cache_name(struct kmem_cache *s)
{
	if (!is_root_cache(s))
		return s->memcg_params->root_cache->name;
	return s->name;
}

static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx)
{
	return s->memcg_params->memcg_caches[idx];
}

static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
{
	if (is_root_cache(s))
		return s;
	return s->memcg_params->root_cache;
}
#else
static inline bool is_root_cache(struct kmem_cache *s)
{
	return true;
}

static inline bool cache_match_memcg(struct kmem_cache *cachep,
				     struct mem_cgroup *memcg)
{
	return true;
}

static inline void memcg_bind_pages(struct kmem_cache *s, int order)
{
}

static inline void memcg_release_pages(struct kmem_cache *s, int order)
{
}

static inline bool slab_equal_or_root(struct kmem_cache *s,
				      struct kmem_cache *p)
{
	return true;
}

static inline const char *cache_name(struct kmem_cache *s)
{
	return s->name;
}

static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx)
{
	return NULL;
}

static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
{
	return s;
}
#endif

static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
{
	struct kmem_cache *cachep;
	struct page *page;

	/*
	 * When kmemcg is not being used, both assignments should return the
	 * same value. but we don't want to pay the assignment price in that
	 * case. If it is not compiled in, the compiler should be smart enough
	 * to not do even the assignment. In that case, slab_equal_or_root
	 * will also be a constant.
	 */
	if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE))
		return s;

	page = virt_to_head_page(x);
	cachep = page->slab_cache;
	if (slab_equal_or_root(cachep, s))
		return cachep;

	pr_err("%s: Wrong slab cache. %s but object is from %s\n",
		__FUNCTION__, cachep->name, s->name);
	WARN_ON_ONCE(1);
	return s;
}
#endif


/*
 * The slab lists for all objects.
 */
struct kmem_cache_node {
	spinlock_t list_lock;

#ifdef CONFIG_SLAB
	struct list_head slabs_partial;	/* partial list first, better asm code */
	struct list_head slabs_full;
	struct list_head slabs_free;
	unsigned long free_objects;
	unsigned int free_limit;
	unsigned int colour_next;	/* Per-node cache coloring */
	struct array_cache *shared;	/* shared per node */
	struct array_cache **alien;	/* on other nodes */
	unsigned long next_reap;	/* updated without locking */
	int free_touched;		/* updated without locking */
#endif

#ifdef CONFIG_SLUB
	unsigned long nr_partial;
	struct list_head partial;
#ifdef CONFIG_SLUB_DEBUG
	atomic_long_t nr_slabs;
	atomic_long_t total_objects;
	struct list_head full;
#endif
#endif

};
Commit	Line	Data
97d06609 CL	1	#ifndef MM_SLAB_H
	2	#define MM_SLAB_H
	3	/*
	4	* Internal slab definitions
	5	*/
	6
	7	/*
	8	* State of the slab allocator.
	9	*
	10	* This is used to describe the states of the allocator during bootup.
	11	* Allocators use this to gradually bootstrap themselves. Most allocators
	12	* have the problem that the structures used for managing slab caches are
	13	* allocated from slab caches themselves.
	14	*/
	15	enum slab_state {
	16	DOWN, /* No slab functionality yet */
	17	PARTIAL, /* SLUB: kmem_cache_node available */
	18	PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */
ce8eb6c4	19	PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
97d06609 CL	20	UP, /* Slab caches usable but not all extras yet */
	21	FULL /* Everything is working */
	22	};
	23
	24	extern enum slab_state slab_state;
	25
18004c5d CL	26	/* The slab cache mutex protects the management structures during changes */
18004c5d CL	27	extern struct mutex slab_mutex;
9b030cb8 CL	28
9b030cb8 CL	29	/* The list of all slab caches on the system */
18004c5d CL	30	extern struct list_head slab_caches;
18004c5d CL	31
9b030cb8 CL	32	/* The slab cache that manages slab cache information */
	33	extern struct kmem_cache *kmem_cache;
	34
45906855 CL	35	unsigned long calculate_alignment(unsigned long flags,
	36	unsigned long align, unsigned long size);
	37
f97d5f63 CL	38	#ifndef CONFIG_SLOB
	39	/* Kmalloc array related functions */
	40	void create_kmalloc_caches(unsigned long);
2c59dd65 CL	41
	42	/* Find the kmalloc slab corresponding for a certain size */
	43	struct kmem_cache *kmalloc_slab(size_t, gfp_t);
f97d5f63 CL	44	#endif
	45
	46
9b030cb8	47	/* Functions provided by the slab allocators */
8a13a4cc	48	extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
97d06609	49
45530c44 CL	50	extern struct kmem_cache create_kmalloc_cache(const char name, size_t size,
	51	unsigned long flags);
	52	extern void create_boot_cache(struct kmem_cache , const char name,
	53	size_t size, unsigned long flags);
	54
2633d7a0	55	struct mem_cgroup;
cbb79694	56	#ifdef CONFIG_SLUB
2633d7a0 GC	57	struct kmem_cache *
	58	__kmem_cache_alias(struct mem_cgroup memcg, const char name, size_t size,
	59	size_t align, unsigned long flags, void (ctor)(void ));
cbb79694	60	#else
2633d7a0 GC	61	static inline struct kmem_cache *
	62	__kmem_cache_alias(struct mem_cgroup memcg, const char name, size_t size,
	63	size_t align, unsigned long flags, void (ctor)(void ))
cbb79694 CL	64	{ return NULL; }
	65	#endif
	66
	67
d8843922 GC	68	/* Legal flag mask for kmem_cache_create(), for various configurations */
	69	#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN \| SLAB_CACHE_DMA \| SLAB_PANIC \| \
	70	SLAB_DESTROY_BY_RCU \| SLAB_DEBUG_OBJECTS )
	71
	72	#if defined(CONFIG_DEBUG_SLAB)
	73	#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER)
	74	#elif defined(CONFIG_SLUB_DEBUG)
	75	#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER \| \
	76	SLAB_TRACE \| SLAB_DEBUG_FREE)
	77	#else
	78	#define SLAB_DEBUG_FLAGS (0)
	79	#endif
	80
	81	#if defined(CONFIG_SLAB)
	82	#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD \| SLAB_NOLEAKTRACE \| \
	83	SLAB_RECLAIM_ACCOUNT \| SLAB_TEMPORARY \| SLAB_NOTRACK)
	84	#elif defined(CONFIG_SLUB)
	85	#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE \| SLAB_RECLAIM_ACCOUNT \| \
	86	SLAB_TEMPORARY \| SLAB_NOTRACK)
	87	#else
	88	#define SLAB_CACHE_FLAGS (0)
	89	#endif
	90
	91	#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS \| SLAB_DEBUG_FLAGS \| SLAB_CACHE_FLAGS)
	92
945cf2b6	93	int __kmem_cache_shutdown(struct kmem_cache *);
945cf2b6	94
b7454ad3 GC	95	struct seq_file;
b7454ad3 GC	96	struct file;
b7454ad3	97
0d7561c6 GC	98	struct slabinfo {
	99	unsigned long active_objs;
	100	unsigned long num_objs;
	101	unsigned long active_slabs;
	102	unsigned long num_slabs;
	103	unsigned long shared_avail;
	104	unsigned int limit;
	105	unsigned int batchcount;
	106	unsigned int shared;
	107	unsigned int objects_per_slab;
	108	unsigned int cache_order;
	109	};
	110
	111	void get_slabinfo(struct kmem_cache s, struct slabinfo sinfo);
	112	void slabinfo_show_stats(struct seq_file m, struct kmem_cache s);
b7454ad3 GC	113	ssize_t slabinfo_write(struct file file, const char __user buffer,
b7454ad3 GC	114	size_t count, loff_t *ppos);
ba6c496e GC	115
	116	#ifdef CONFIG_MEMCG_KMEM
	117	static inline bool is_root_cache(struct kmem_cache *s)
	118	{
	119	return !s->memcg_params \|\| s->memcg_params->is_root_cache;
	120	}
2633d7a0 GC	121
	122	static inline bool cache_match_memcg(struct kmem_cache *cachep,
	123	struct mem_cgroup *memcg)
	124	{
	125	return (is_root_cache(cachep) && !memcg) \|\|
	126	(cachep->memcg_params->memcg == memcg);
	127	}
b9ce5ef4	128
1f458cbf GC	129	static inline void memcg_bind_pages(struct kmem_cache *s, int order)
	130	{
	131	if (!is_root_cache(s))
	132	atomic_add(1 << order, &s->memcg_params->nr_pages);
	133	}
	134
	135	static inline void memcg_release_pages(struct kmem_cache *s, int order)
	136	{
	137	if (is_root_cache(s))
	138	return;
	139
	140	if (atomic_sub_and_test((1 << order), &s->memcg_params->nr_pages))
	141	mem_cgroup_destroy_cache(s);
	142	}
	143
b9ce5ef4 GC	144	static inline bool slab_equal_or_root(struct kmem_cache *s,
	145	struct kmem_cache *p)
	146	{
	147	return (p == s) \|\|
	148	(s->memcg_params && (p == s->memcg_params->root_cache));
	149	}
749c5415 GC	150
	151	/*
	152	* We use suffixes to the name in memcg because we can't have caches
	153	* created in the system with the same name. But when we print them
	154	* locally, better refer to them with the base name
	155	*/
	156	static inline const char cache_name(struct kmem_cache s)
	157	{
	158	if (!is_root_cache(s))
	159	return s->memcg_params->root_cache->name;
	160	return s->name;
	161	}
	162
	163	static inline struct kmem_cache cache_from_memcg(struct kmem_cache s, int idx)
	164	{
	165	return s->memcg_params->memcg_caches[idx];
	166	}
943a451a GC	167
	168	static inline struct kmem_cache memcg_root_cache(struct kmem_cache s)
	169	{
	170	if (is_root_cache(s))
	171	return s;
	172	return s->memcg_params->root_cache;
	173	}
ba6c496e GC	174	#else
	175	static inline bool is_root_cache(struct kmem_cache *s)
	176	{
	177	return true;
	178	}
	179
2633d7a0 GC	180	static inline bool cache_match_memcg(struct kmem_cache *cachep,
	181	struct mem_cgroup *memcg)
	182	{
	183	return true;
	184	}
b9ce5ef4	185
1f458cbf GC	186	static inline void memcg_bind_pages(struct kmem_cache *s, int order)
	187	{
	188	}
	189
	190	static inline void memcg_release_pages(struct kmem_cache *s, int order)
	191	{
	192	}
	193
b9ce5ef4 GC	194	static inline bool slab_equal_or_root(struct kmem_cache *s,
	195	struct kmem_cache *p)
	196	{
	197	return true;
	198	}
749c5415 GC	199
	200	static inline const char cache_name(struct kmem_cache s)
	201	{
	202	return s->name;
	203	}
	204
	205	static inline struct kmem_cache cache_from_memcg(struct kmem_cache s, int idx)
	206	{
	207	return NULL;
	208	}
943a451a GC	209
	210	static inline struct kmem_cache memcg_root_cache(struct kmem_cache s)
	211	{
	212	return s;
	213	}
ba6c496e	214	#endif
b9ce5ef4 GC	215
	216	static inline struct kmem_cache cache_from_obj(struct kmem_cache s, void *x)
	217	{
	218	struct kmem_cache *cachep;
	219	struct page *page;
	220
	221	/*
	222	* When kmemcg is not being used, both assignments should return the
	223	* same value. but we don't want to pay the assignment price in that
	224	* case. If it is not compiled in, the compiler should be smart enough
	225	* to not do even the assignment. In that case, slab_equal_or_root
	226	* will also be a constant.
	227	*/
	228	if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE))
	229	return s;
	230
	231	page = virt_to_head_page(x);
	232	cachep = page->slab_cache;
	233	if (slab_equal_or_root(cachep, s))
	234	return cachep;
	235
	236	pr_err("%s: Wrong slab cache. %s but object is from %s\n",
	237	__FUNCTION__, cachep->name, s->name);
	238	WARN_ON_ONCE(1);
	239	return s;
	240	}
97d06609	241	#endif
ca34956b CL	242
	243
	244	/*
	245	* The slab lists for all objects.
	246	*/
	247	struct kmem_cache_node {
	248	spinlock_t list_lock;
	249
	250	#ifdef CONFIG_SLAB
	251	struct list_head slabs_partial; /* partial list first, better asm code */
	252	struct list_head slabs_full;
	253	struct list_head slabs_free;
	254	unsigned long free_objects;
	255	unsigned int free_limit;
	256	unsigned int colour_next; /* Per-node cache coloring */
	257	struct array_cache shared; / shared per node */
	258	struct array_cache *alien; / on other nodes */
	259	unsigned long next_reap; /* updated without locking */
	260	int free_touched; /* updated without locking */
	261	#endif
	262
	263	#ifdef CONFIG_SLUB
	264	unsigned long nr_partial;
	265	struct list_head partial;
	266	#ifdef CONFIG_SLUB_DEBUG
	267	atomic_long_t nr_slabs;
	268	atomic_long_t total_objects;
	269	struct list_head full;
	270	#endif
	271	#endif
	272
	273	};