[linux.git] / include / linux / nodemask.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_NODEMASK_H
#define __LINUX_NODEMASK_H

/*
 * Nodemasks provide a bitmap suitable for representing the
 * set of Node's in a system, one bit position per Node number.
 *
 * See detailed comments in the file linux/bitmap.h describing the
 * data type on which these nodemasks are based.
 *
 * For details of nodemask_parse_user(), see bitmap_parse_user() in
 * lib/bitmap.c.  For details of nodelist_parse(), see bitmap_parselist(),
 * also in bitmap.c.  For details of node_remap(), see bitmap_bitremap in
 * lib/bitmap.c.  For details of nodes_remap(), see bitmap_remap in
 * lib/bitmap.c.  For details of nodes_onto(), see bitmap_onto in
 * lib/bitmap.c.  For details of nodes_fold(), see bitmap_fold in
 * lib/bitmap.c.
 *
 * The available nodemask operations are:
 *
 * void node_set(node, mask)		turn on bit 'node' in mask
 * void node_clear(node, mask)		turn off bit 'node' in mask
 * void nodes_setall(mask)		set all bits
 * void nodes_clear(mask)		clear all bits
 * int node_isset(node, mask)		true iff bit 'node' set in mask
 * int node_test_and_set(node, mask)	test and set bit 'node' in mask
 *
 * void nodes_and(dst, src1, src2)	dst = src1 & src2  [intersection]
 * void nodes_or(dst, src1, src2)	dst = src1 | src2  [union]
 * void nodes_xor(dst, src1, src2)	dst = src1 ^ src2
 * void nodes_andnot(dst, src1, src2)	dst = src1 & ~src2
 * void nodes_complement(dst, src)	dst = ~src
 *
 * int nodes_equal(mask1, mask2)	Does mask1 == mask2?
 * int nodes_intersects(mask1, mask2)	Do mask1 and mask2 intersect?
 * int nodes_subset(mask1, mask2)	Is mask1 a subset of mask2?
 * int nodes_empty(mask)		Is mask empty (no bits sets)?
 * int nodes_full(mask)			Is mask full (all bits sets)?
 * int nodes_weight(mask)		Hamming weight - number of set bits
 *
 * void nodes_shift_right(dst, src, n)	Shift right
 * void nodes_shift_left(dst, src, n)	Shift left
 *
 * unsigned int first_node(mask)	Number lowest set bit, or MAX_NUMNODES
 * unsigend int next_node(node, mask)	Next node past 'node', or MAX_NUMNODES
 * unsigned int next_node_in(node, mask) Next node past 'node', or wrap to first,
 *					or MAX_NUMNODES
 * unsigned int first_unset_node(mask)	First node not set in mask, or
 *					MAX_NUMNODES
 *
 * nodemask_t nodemask_of_node(node)	Return nodemask with bit 'node' set
 * NODE_MASK_ALL			Initializer - all bits set
 * NODE_MASK_NONE			Initializer - no bits set
 * unsigned long *nodes_addr(mask)	Array of unsigned long's in mask
 *
 * int nodemask_parse_user(ubuf, ulen, mask)	Parse ascii string as nodemask
 * int nodelist_parse(buf, map)		Parse ascii string as nodelist
 * int node_remap(oldbit, old, new)	newbit = map(old, new)(oldbit)
 * void nodes_remap(dst, src, old, new)	*dst = map(old, new)(src)
 * void nodes_onto(dst, orig, relmap)	*dst = orig relative to relmap
 * void nodes_fold(dst, orig, sz)	dst bits = orig bits mod sz
 *
 * for_each_node_mask(node, mask)	for-loop node over mask
 *
 * int num_online_nodes()		Number of online Nodes
 * int num_possible_nodes()		Number of all possible Nodes
 *
 * int node_random(mask)		Random node with set bit in mask
 *
 * int node_online(node)		Is some node online?
 * int node_possible(node)		Is some node possible?
 *
 * node_set_online(node)		set bit 'node' in node_online_map
 * node_set_offline(node)		clear bit 'node' in node_online_map
 *
 * for_each_node(node)			for-loop node over node_possible_map
 * for_each_online_node(node)		for-loop node over node_online_map
 *
 * Subtlety:
 * 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
 *    to generate slightly worse code.  So use a simple one-line #define
 *    for node_isset(), instead of wrapping an inline inside a macro, the
 *    way we do the other calls.
 *
 * NODEMASK_SCRATCH
 * When doing above logical AND, OR, XOR, Remap operations the callers tend to
 * need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
 * nodemask_t's consume too much stack space.  NODEMASK_SCRATCH is a helper
 * for such situations. See below and CPUMASK_ALLOC also.
 */

#include <linux/threads.h>
#include <linux/bitmap.h>
#include <linux/minmax.h>
#include <linux/numa.h>
#include <linux/random.h>

typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
extern nodemask_t _unused_nodemask_arg_;

/**
 * nodemask_pr_args - printf args to output a nodemask
 * @maskp: nodemask to be printed
 *
 * Can be used to provide arguments for '%*pb[l]' when printing a nodemask.
 */
#define nodemask_pr_args(maskp)	__nodemask_pr_numnodes(maskp), \
				__nodemask_pr_bits(maskp)
static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
{
	return m ? MAX_NUMNODES : 0;
}
static inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
{
	return m ? m->bits : NULL;
}

/*
 * The inline keyword gives the compiler room to decide to inline, or
 * not inline a function as it sees best.  However, as these functions
 * are called in both __init and non-__init functions, if they are not
 * inlined we will end up with a section mismatch error (of the type of
 * freeable items not being freed).  So we must use __always_inline here
 * to fix the problem.  If other functions in the future also end up in
 * this situation they will also need to be annotated as __always_inline
 */
#define node_set(node, dst) __node_set((node), &(dst))
static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
{
	set_bit(node, dstp->bits);
}

#define node_clear(node, dst) __node_clear((node), &(dst))
static inline void __node_clear(int node, volatile nodemask_t *dstp)
{
	clear_bit(node, dstp->bits);
}

#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
{
	bitmap_fill(dstp->bits, nbits);
}

#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
{
	bitmap_zero(dstp->bits, nbits);
}

/* No static inline type checking - see Subtlety (1) above. */
#define node_isset(node, nodemask) test_bit((node), (nodemask).bits)

#define node_test_and_set(node, nodemask) \
			__node_test_and_set((node), &(nodemask))
static inline bool __node_test_and_set(int node, nodemask_t *addr)
{
	return test_and_set_bit(node, addr->bits);
}

#define nodes_and(dst, src1, src2) \
			__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_or(dst, src1, src2) \
			__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_xor(dst, src1, src2) \
			__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_andnot(dst, src1, src2) \
			__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_complement(dst, src) \
			__nodes_complement(&(dst), &(src), MAX_NUMNODES)
static inline void __nodes_complement(nodemask_t *dstp,
					const nodemask_t *srcp, unsigned int nbits)
{
	bitmap_complement(dstp->bits, srcp->bits, nbits);
}

#define nodes_equal(src1, src2) \
			__nodes_equal(&(src1), &(src2), MAX_NUMNODES)
static inline bool __nodes_equal(const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	return bitmap_equal(src1p->bits, src2p->bits, nbits);
}

#define nodes_intersects(src1, src2) \
			__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
static inline bool __nodes_intersects(const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
}

#define nodes_subset(src1, src2) \
			__nodes_subset(&(src1), &(src2), MAX_NUMNODES)
static inline bool __nodes_subset(const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	return bitmap_subset(src1p->bits, src2p->bits, nbits);
}

#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
{
	return bitmap_empty(srcp->bits, nbits);
}

#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
{
	return bitmap_full(srcp->bits, nbits);
}

#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
{
	return bitmap_weight(srcp->bits, nbits);
}

#define nodes_shift_right(dst, src, n) \
			__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
static inline void __nodes_shift_right(nodemask_t *dstp,
					const nodemask_t *srcp, int n, int nbits)
{
	bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
}

#define nodes_shift_left(dst, src, n) \
			__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
static inline void __nodes_shift_left(nodemask_t *dstp,
					const nodemask_t *srcp, int n, int nbits)
{
	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
}

/* FIXME: better would be to fix all architectures to never return
          > MAX_NUMNODES, then the silly min_ts could be dropped. */

#define first_node(src) __first_node(&(src))
static inline unsigned int __first_node(const nodemask_t *srcp)
{
	return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
}

#define next_node(n, src) __next_node((n), &(src))
static inline unsigned int __next_node(int n, const nodemask_t *srcp)
{
	return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
}

/*
 * Find the next present node in src, starting after node n, wrapping around to
 * the first node in src if needed.  Returns MAX_NUMNODES if src is empty.
 */
#define next_node_in(n, src) __next_node_in((n), &(src))
static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
{
	unsigned int ret = __next_node(node, srcp);

	if (ret == MAX_NUMNODES)
		ret = __first_node(srcp);
	return ret;
}

static inline void init_nodemask_of_node(nodemask_t *mask, int node)
{
	nodes_clear(*mask);
	node_set(node, *mask);
}

#define nodemask_of_node(node)						\
({									\
	typeof(_unused_nodemask_arg_) m;				\
	if (sizeof(m) == sizeof(unsigned long)) {			\
		m.bits[0] = 1UL << (node);				\
	} else {							\
		init_nodemask_of_node(&m, (node));			\
	}								\
	m;								\
})

#define first_unset_node(mask) __first_unset_node(&(mask))
static inline unsigned int __first_unset_node(const nodemask_t *maskp)
{
	return min_t(unsigned int, MAX_NUMNODES,
			find_first_zero_bit(maskp->bits, MAX_NUMNODES));
}

#define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)

#if MAX_NUMNODES <= BITS_PER_LONG

#define NODE_MASK_ALL							\
((nodemask_t) { {							\
	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD		\
} })

#else

#define NODE_MASK_ALL							\
((nodemask_t) { {							\
	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL,			\
	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD		\
} })

#endif

#define NODE_MASK_NONE							\
((nodemask_t) { {							\
	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] =  0UL			\
} })

#define nodes_addr(src) ((src).bits)

#define nodemask_parse_user(ubuf, ulen, dst) \
		__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
static inline int __nodemask_parse_user(const char __user *buf, int len,
					nodemask_t *dstp, int nbits)
{
	return bitmap_parse_user(buf, len, dstp->bits, nbits);
}

#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
{
	return bitmap_parselist(buf, dstp->bits, nbits);
}

#define node_remap(oldbit, old, new) \
		__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
static inline int __node_remap(int oldbit,
		const nodemask_t *oldp, const nodemask_t *newp, int nbits)
{
	return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
}

#define nodes_remap(dst, src, old, new) \
		__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
		const nodemask_t *oldp, const nodemask_t *newp, int nbits)
{
	bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
}

#define nodes_onto(dst, orig, relmap) \
		__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
		const nodemask_t *relmapp, int nbits)
{
	bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
}

#define nodes_fold(dst, orig, sz) \
		__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
		int sz, int nbits)
{
	bitmap_fold(dstp->bits, origp->bits, sz, nbits);
}

#if MAX_NUMNODES > 1
#define for_each_node_mask(node, mask)				    \
	for ((node) = first_node(mask);				    \
	     (node >= 0) && (node) < MAX_NUMNODES;		    \
	     (node) = next_node((node), (mask)))
#else /* MAX_NUMNODES == 1 */
#define for_each_node_mask(node, mask)                                  \
	for ((node) = 0; (node) < 1 && !nodes_empty(mask); (node)++)
#endif /* MAX_NUMNODES */

/*
 * Bitmasks that are kept for all the nodes.
 */
enum node_states {
	N_POSSIBLE,		/* The node could become online at some point */
	N_ONLINE,		/* The node is online */
	N_NORMAL_MEMORY,	/* The node has regular memory */
#ifdef CONFIG_HIGHMEM
	N_HIGH_MEMORY,		/* The node has regular or high memory */
#else
	N_HIGH_MEMORY = N_NORMAL_MEMORY,
#endif
	N_MEMORY,		/* The node has memory(regular, high, movable) */
	N_CPU,		/* The node has one or more cpus */
	N_GENERIC_INITIATOR,	/* The node has one or more Generic Initiators */
	NR_NODE_STATES
};

/*
 * The following particular system nodemasks and operations
 * on them manage all possible and online nodes.
 */

extern nodemask_t node_states[NR_NODE_STATES];

#if MAX_NUMNODES > 1
static inline int node_state(int node, enum node_states state)
{
	return node_isset(node, node_states[state]);
}

static inline void node_set_state(int node, enum node_states state)
{
	__node_set(node, &node_states[state]);
}

static inline void node_clear_state(int node, enum node_states state)
{
	__node_clear(node, &node_states[state]);
}

static inline int num_node_state(enum node_states state)
{
	return nodes_weight(node_states[state]);
}

#define for_each_node_state(__node, __state) \
	for_each_node_mask((__node), node_states[__state])

#define first_online_node	first_node(node_states[N_ONLINE])
#define first_memory_node	first_node(node_states[N_MEMORY])
static inline unsigned int next_online_node(int nid)
{
	return next_node(nid, node_states[N_ONLINE]);
}
static inline unsigned int next_memory_node(int nid)
{
	return next_node(nid, node_states[N_MEMORY]);
}

extern unsigned int nr_node_ids;
extern unsigned int nr_online_nodes;

static inline void node_set_online(int nid)
{
	node_set_state(nid, N_ONLINE);
	nr_online_nodes = num_node_state(N_ONLINE);
}

static inline void node_set_offline(int nid)
{
	node_clear_state(nid, N_ONLINE);
	nr_online_nodes = num_node_state(N_ONLINE);
}

#else

static inline int node_state(int node, enum node_states state)
{
	return node == 0;
}

static inline void node_set_state(int node, enum node_states state)
{
}

static inline void node_clear_state(int node, enum node_states state)
{
}

static inline int num_node_state(enum node_states state)
{
	return 1;
}

#define for_each_node_state(node, __state) \
	for ( (node) = 0; (node) == 0; (node) = 1)

#define first_online_node	0
#define first_memory_node	0
#define next_online_node(nid)	(MAX_NUMNODES)
#define nr_node_ids		1U
#define nr_online_nodes		1U

#define node_set_online(node)	   node_set_state((node), N_ONLINE)
#define node_set_offline(node)	   node_clear_state((node), N_ONLINE)

#endif

static inline int node_random(const nodemask_t *maskp)
{
#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
	int w, bit = NUMA_NO_NODE;

	w = nodes_weight(*maskp);
	if (w)
		bit = bitmap_ord_to_pos(maskp->bits,
			get_random_int() % w, MAX_NUMNODES);
	return bit;
#else
	return 0;
#endif
}

#define node_online_map 	node_states[N_ONLINE]
#define node_possible_map 	node_states[N_POSSIBLE]

#define num_online_nodes()	num_node_state(N_ONLINE)
#define num_possible_nodes()	num_node_state(N_POSSIBLE)
#define node_online(node)	node_state((node), N_ONLINE)
#define node_possible(node)	node_state((node), N_POSSIBLE)

#define for_each_node(node)	   for_each_node_state(node, N_POSSIBLE)
#define for_each_online_node(node) for_each_node_state(node, N_ONLINE)

/*
 * For nodemask scratch area.
 * NODEMASK_ALLOC(type, name) allocates an object with a specified type and
 * name.
 */
#if NODES_SHIFT > 8 /* nodemask_t > 32 bytes */
#define NODEMASK_ALLOC(type, name, gfp_flags)	\
			type *name = kmalloc(sizeof(*name), gfp_flags)
#define NODEMASK_FREE(m)			kfree(m)
#else
#define NODEMASK_ALLOC(type, name, gfp_flags)	type _##name, *name = &_##name
#define NODEMASK_FREE(m)			do {} while (0)
#endif

/* Example structure for using NODEMASK_ALLOC, used in mempolicy. */
struct nodemask_scratch {
	nodemask_t	mask1;
	nodemask_t	mask2;
};

#define NODEMASK_SCRATCH(x)						\
			NODEMASK_ALLOC(struct nodemask_scratch, x,	\
					GFP_KERNEL | __GFP_NORETRY)
#define NODEMASK_SCRATCH_FREE(x)	NODEMASK_FREE(x)


#endif /* __LINUX_NODEMASK_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	#ifndef __LINUX_NODEMASK_H
	3	#define __LINUX_NODEMASK_H
	4
	5	/*
	6	* Nodemasks provide a bitmap suitable for representing the
	7	* set of Node's in a system, one bit position per Node number.
	8	*
	9	* See detailed comments in the file linux/bitmap.h describing the
	10	* data type on which these nodemasks are based.
	11	*
46385326 TH	12	* For details of nodemask_parse_user(), see bitmap_parse_user() in
	13	* lib/bitmap.c. For details of nodelist_parse(), see bitmap_parselist(),
	14	* also in bitmap.c. For details of node_remap(), see bitmap_bitremap in
	15	* lib/bitmap.c. For details of nodes_remap(), see bitmap_remap in
	16	* lib/bitmap.c. For details of nodes_onto(), see bitmap_onto in
	17	* lib/bitmap.c. For details of nodes_fold(), see bitmap_fold in
	18	* lib/bitmap.c.
1da177e4 LT	19	*
	20	* The available nodemask operations are:
	21	*
	22	* void node_set(node, mask) turn on bit 'node' in mask
	23	* void node_clear(node, mask) turn off bit 'node' in mask
	24	* void nodes_setall(mask) set all bits
	25	* void nodes_clear(mask) clear all bits
	26	* int node_isset(node, mask) true iff bit 'node' set in mask
	27	* int node_test_and_set(node, mask) test and set bit 'node' in mask
	28	*
	29	* void nodes_and(dst, src1, src2) dst = src1 & src2 [intersection]
	30	* void nodes_or(dst, src1, src2) dst = src1 \| src2 [union]
	31	* void nodes_xor(dst, src1, src2) dst = src1 ^ src2
	32	* void nodes_andnot(dst, src1, src2) dst = src1 & ~src2
	33	* void nodes_complement(dst, src) dst = ~src
	34	*
	35	* int nodes_equal(mask1, mask2) Does mask1 == mask2?
	36	* int nodes_intersects(mask1, mask2) Do mask1 and mask2 intersect?
	37	* int nodes_subset(mask1, mask2) Is mask1 a subset of mask2?
	38	* int nodes_empty(mask) Is mask empty (no bits sets)?
	39	* int nodes_full(mask) Is mask full (all bits sets)?
	40	* int nodes_weight(mask) Hamming weight - number of set bits
	41	*
	42	* void nodes_shift_right(dst, src, n) Shift right
	43	* void nodes_shift_left(dst, src, n) Shift left
	44	*
0dfe5407 KC	45	* unsigned int first_node(mask) Number lowest set bit, or MAX_NUMNODES
	46	* unsigend int next_node(node, mask) Next node past 'node', or MAX_NUMNODES
	47	* unsigned int next_node_in(node, mask) Next node past 'node', or wrap to first,
0edaf86c	48	* or MAX_NUMNODES
0dfe5407	49	* unsigned int first_unset_node(mask) First node not set in mask, or
0edaf86c	50	* MAX_NUMNODES
1da177e4 LT	51	*
	52	* nodemask_t nodemask_of_node(node) Return nodemask with bit 'node' set
	53	* NODE_MASK_ALL Initializer - all bits set
	54	* NODE_MASK_NONE Initializer - no bits set
	55	* unsigned long *nodes_addr(mask) Array of unsigned long's in mask
	56	*
01a3ee2b	57	* int nodemask_parse_user(ubuf, ulen, mask) Parse ascii string as nodemask
1da177e4	58	* int nodelist_parse(buf, map) Parse ascii string as nodelist
fb5eeeee	59	* int node_remap(oldbit, old, new) newbit = map(old, new)(oldbit)
7ea931c9 PJ	60	* void nodes_remap(dst, src, old, new) *dst = map(old, new)(src)
	61	* void nodes_onto(dst, orig, relmap) *dst = orig relative to relmap
	62	* void nodes_fold(dst, orig, sz) dst bits = orig bits mod sz
1da177e4 LT	63	*
	64	* for_each_node_mask(node, mask) for-loop node over mask
	65	*
	66	* int num_online_nodes() Number of online Nodes
	67	* int num_possible_nodes() Number of all possible Nodes
	68	*
778d3b0f MH	69	* int node_random(mask) Random node with set bit in mask
778d3b0f MH	70	*
1da177e4 LT	71	* int node_online(node) Is some node online?
	72	* int node_possible(node) Is some node possible?
	73	*
1da177e4 LT	74	* node_set_online(node) set bit 'node' in node_online_map
	75	* node_set_offline(node) clear bit 'node' in node_online_map
	76	*
	77	* for_each_node(node) for-loop node over node_possible_map
	78	* for_each_online_node(node) for-loop node over node_online_map
	79	*
	80	* Subtlety:
	81	* 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
	82	* to generate slightly worse code. So use a simple one-line #define
	83	* for node_isset(), instead of wrapping an inline inside a macro, the
	84	* way we do the other calls.
4bfc4495 KH	85	*
	86	* NODEMASK_SCRATCH
	87	* When doing above logical AND, OR, XOR, Remap operations the callers tend to
	88	* need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
	89	* nodemask_t's consume too much stack space. NODEMASK_SCRATCH is a helper
	90	* for such situations. See below and CPUMASK_ALLOC also.
1da177e4 LT	91	*/
1da177e4 LT	92
1da177e4 LT	93	#include <linux/threads.h>
1da177e4 LT	94	#include <linux/bitmap.h>
b296a6d5	95	#include <linux/minmax.h>
1da177e4	96	#include <linux/numa.h>
36d4b36b	97	#include <linux/random.h>
1da177e4 LT	98
	99	typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
	100	extern nodemask_t _unused_nodemask_arg_;
	101
f1bbc032 TH	102	/**
	103	* nodemask_pr_args - printf args to output a nodemask
	104	* @maskp: nodemask to be printed
	105	*
	106	* Can be used to provide arguments for '%*pb[l]' when printing a nodemask.
	107	*/
1334be36 AB	108	#define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \
	109	__nodemask_pr_bits(maskp)
	110	static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
	111	{
	112	return m ? MAX_NUMNODES : 0;
	113	}
	114	static inline const unsigned long __nodemask_pr_bits(const nodemask_t m)
	115	{
	116	return m ? m->bits : NULL;
	117	}
f1bbc032	118
323f54ed TR	119	/*
	120	* The inline keyword gives the compiler room to decide to inline, or
	121	* not inline a function as it sees best. However, as these functions
	122	* are called in both __init and non-__init functions, if they are not
c23c8082	123	* inlined we will end up with a section mismatch error (of the type of
323f54ed TR	124	* freeable items not being freed). So we must use __always_inline here
	125	* to fix the problem. If other functions in the future also end up in
	126	* this situation they will also need to be annotated as __always_inline
	127	*/
1da177e4	128	#define node_set(node, dst) __node_set((node), &(dst))
323f54ed	129	static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
1da177e4 LT	130	{
	131	set_bit(node, dstp->bits);
	132	}
	133
	134	#define node_clear(node, dst) __node_clear((node), &(dst))
	135	static inline void __node_clear(int node, volatile nodemask_t *dstp)
	136	{
	137	clear_bit(node, dstp->bits);
	138	}
	139
	140	#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
33c4fa8c	141	static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
1da177e4 LT	142	{
	143	bitmap_fill(dstp->bits, nbits);
	144	}
	145
	146	#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
33c4fa8c	147	static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
1da177e4 LT	148	{
	149	bitmap_zero(dstp->bits, nbits);
	150	}
	151
	152	/* No static inline type checking - see Subtlety (1) above. */
	153	#define node_isset(node, nodemask) test_bit((node), (nodemask).bits)
	154
	155	#define node_test_and_set(node, nodemask) \
	156	__node_test_and_set((node), &(nodemask))
0dfe5407	157	static inline bool __node_test_and_set(int node, nodemask_t *addr)
1da177e4 LT	158	{
	159	return test_and_set_bit(node, addr->bits);
	160	}
	161
	162	#define nodes_and(dst, src1, src2) \
	163	__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
	164	static inline void __nodes_and(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	165	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	166	{
	167	bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
	168	}
	169
	170	#define nodes_or(dst, src1, src2) \
	171	__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
	172	static inline void __nodes_or(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	173	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	174	{
	175	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
	176	}
	177
	178	#define nodes_xor(dst, src1, src2) \
	179	__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
	180	static inline void __nodes_xor(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	181	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	182	{
	183	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
	184	}
	185
	186	#define nodes_andnot(dst, src1, src2) \
	187	__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
	188	static inline void __nodes_andnot(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	189	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	190	{
	191	bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
	192	}
	193
	194	#define nodes_complement(dst, src) \
	195	__nodes_complement(&(dst), &(src), MAX_NUMNODES)
	196	static inline void __nodes_complement(nodemask_t *dstp,
33c4fa8c	197	const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	198	{
	199	bitmap_complement(dstp->bits, srcp->bits, nbits);
	200	}
	201
	202	#define nodes_equal(src1, src2) \
	203	__nodes_equal(&(src1), &(src2), MAX_NUMNODES)
0dfe5407	204	static inline bool __nodes_equal(const nodemask_t *src1p,
33c4fa8c	205	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	206	{
	207	return bitmap_equal(src1p->bits, src2p->bits, nbits);
	208	}
	209
	210	#define nodes_intersects(src1, src2) \
	211	__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
0dfe5407	212	static inline bool __nodes_intersects(const nodemask_t *src1p,
33c4fa8c	213	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	214	{
	215	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
	216	}
	217
	218	#define nodes_subset(src1, src2) \
	219	__nodes_subset(&(src1), &(src2), MAX_NUMNODES)
0dfe5407	220	static inline bool __nodes_subset(const nodemask_t *src1p,
33c4fa8c	221	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	222	{
	223	return bitmap_subset(src1p->bits, src2p->bits, nbits);
	224	}
	225
	226	#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
0dfe5407	227	static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	228	{
	229	return bitmap_empty(srcp->bits, nbits);
	230	}
	231
	232	#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
0dfe5407	233	static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	234	{
	235	return bitmap_full(srcp->bits, nbits);
	236	}
	237
	238	#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
33c4fa8c	239	static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	240	{
	241	return bitmap_weight(srcp->bits, nbits);
	242	}
	243
	244	#define nodes_shift_right(dst, src, n) \
	245	__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
	246	static inline void __nodes_shift_right(nodemask_t *dstp,
	247	const nodemask_t *srcp, int n, int nbits)
	248	{
	249	bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
	250	}
	251
	252	#define nodes_shift_left(dst, src, n) \
	253	__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
	254	static inline void __nodes_shift_left(nodemask_t *dstp,
	255	const nodemask_t *srcp, int n, int nbits)
	256	{
	257	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
	258	}
	259
	260	/* FIXME: better would be to fix all architectures to never return
	261	> MAX_NUMNODES, then the silly min_ts could be dropped. */
	262
	263	#define first_node(src) __first_node(&(src))
0dfe5407	264	static inline unsigned int __first_node(const nodemask_t *srcp)
1da177e4	265	{
0dfe5407	266	return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
1da177e4 LT	267	}
	268
	269	#define next_node(n, src) __next_node((n), &(src))
0dfe5407	270	static inline unsigned int __next_node(int n, const nodemask_t *srcp)
1da177e4	271	{
0dfe5407	272	return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
1da177e4 LT	273	}
1da177e4 LT	274
0edaf86c AM	275	/*
	276	* Find the next present node in src, starting after node n, wrapping around to
	277	* the first node in src if needed. Returns MAX_NUMNODES if src is empty.
	278	*/
	279	#define next_node_in(n, src) __next_node_in((n), &(src))
36d4b36b YN	280	static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
	281	{
	282	unsigned int ret = __next_node(node, srcp);
	283
	284	if (ret == MAX_NUMNODES)
	285	ret = __first_node(srcp);
	286	return ret;
	287	}
0edaf86c	288
c1e6c8d0 LS	289	static inline void init_nodemask_of_node(nodemask_t *mask, int node)
	290	{
	291	nodes_clear(*mask);
	292	node_set(node, *mask);
	293	}
	294
1da177e4 LT	295	#define nodemask_of_node(node) \
	296	({ \
	297	typeof(_unused_nodemask_arg_) m; \
	298	if (sizeof(m) == sizeof(unsigned long)) { \
c1e6c8d0	299	m.bits[0] = 1UL << (node); \
1da177e4	300	} else { \
c1e6c8d0	301	init_nodemask_of_node(&m, (node)); \
1da177e4 LT	302	} \
	303	m; \
	304	})
	305
	306	#define first_unset_node(mask) __first_unset_node(&(mask))
0dfe5407	307	static inline unsigned int __first_unset_node(const nodemask_t *maskp)
1da177e4	308	{
0dfe5407	309	return min_t(unsigned int, MAX_NUMNODES,
1da177e4 LT	310	find_first_zero_bit(maskp->bits, MAX_NUMNODES));
	311	}
	312
	313	#define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)
	314
	315	#if MAX_NUMNODES <= BITS_PER_LONG
	316
	317	#define NODE_MASK_ALL \
	318	((nodemask_t) { { \
	319	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
	320	} })
	321
	322	#else
	323
	324	#define NODE_MASK_ALL \
	325	((nodemask_t) { { \
	326	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL, \
	327	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
	328	} })
	329
	330	#endif
	331
	332	#define NODE_MASK_NONE \
	333	((nodemask_t) { { \
	334	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] = 0UL \
	335	} })
	336
	337	#define nodes_addr(src) ((src).bits)
	338
01a3ee2b RC	339	#define nodemask_parse_user(ubuf, ulen, dst) \
	340	__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
	341	static inline int __nodemask_parse_user(const char __user *buf, int len,
1da177e4 LT	342	nodemask_t *dstp, int nbits)
1da177e4 LT	343	{
01a3ee2b	344	return bitmap_parse_user(buf, len, dstp->bits, nbits);
1da177e4 LT	345	}
1da177e4 LT	346
1da177e4 LT	347	#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
	348	static inline int __nodelist_parse(const char buf, nodemask_t dstp, int nbits)
	349	{
	350	return bitmap_parselist(buf, dstp->bits, nbits);
	351	}
	352
fb5eeeee PJ	353	#define node_remap(oldbit, old, new) \
	354	__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
	355	static inline int __node_remap(int oldbit,
	356	const nodemask_t oldp, const nodemask_t newp, int nbits)
	357	{
	358	return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
	359	}
	360
	361	#define nodes_remap(dst, src, old, new) \
	362	__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
	363	static inline void __nodes_remap(nodemask_t dstp, const nodemask_t srcp,
	364	const nodemask_t oldp, const nodemask_t newp, int nbits)
	365	{
	366	bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
	367	}
	368
7ea931c9 PJ	369	#define nodes_onto(dst, orig, relmap) \
	370	__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
	371	static inline void __nodes_onto(nodemask_t dstp, const nodemask_t origp,
	372	const nodemask_t *relmapp, int nbits)
	373	{
	374	bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
	375	}
	376
	377	#define nodes_fold(dst, orig, sz) \
	378	__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
	379	static inline void __nodes_fold(nodemask_t dstp, const nodemask_t origp,
	380	int sz, int nbits)
	381	{
	382	bitmap_fold(dstp->bits, origp->bits, sz, nbits);
	383	}
	384
1da177e4	385	#if MAX_NUMNODES > 1
37462a92 CD	386	#define for_each_node_mask(node, mask) \
	387	for ((node) = first_node(mask); \
	388	(node >= 0) && (node) < MAX_NUMNODES; \
	389	(node) = next_node((node), (mask)))
1da177e4	390	#else /* MAX_NUMNODES == 1 */
37462a92 CD	391	#define for_each_node_mask(node, mask) \
37462a92 CD	392	for ((node) = 0; (node) < 1 && !nodes_empty(mask); (node)++)
1da177e4 LT	393	#endif /* MAX_NUMNODES */
1da177e4 LT	394
13808910 CL	395	/*
	396	* Bitmasks that are kept for all the nodes.
	397	*/
	398	enum node_states {
7ea1530a CL	399	N_POSSIBLE, /* The node could become online at some point */
	400	N_ONLINE, /* The node is online */
	401	N_NORMAL_MEMORY, /* The node has regular memory */
	402	#ifdef CONFIG_HIGHMEM
	403	N_HIGH_MEMORY, /* The node has regular or high memory */
	404	#else
	405	N_HIGH_MEMORY = N_NORMAL_MEMORY,
	406	#endif
20b2f52b	407	N_MEMORY, /* The node has memory(regular, high, movable) */
37c0708d	408	N_CPU, /* The node has one or more cpus */
894c26a1	409	N_GENERIC_INITIATOR, /* The node has one or more Generic Initiators */
13808910 CL	410	NR_NODE_STATES
	411	};
	412
1da177e4 LT	413	/*
	414	* The following particular system nodemasks and operations
	415	* on them manage all possible and online nodes.
	416	*/
	417
13808910	418	extern nodemask_t node_states[NR_NODE_STATES];
1da177e4 LT	419
1da177e4 LT	420	#if MAX_NUMNODES > 1
13808910 CL	421	static inline int node_state(int node, enum node_states state)
	422	{
	423	return node_isset(node, node_states[state]);
	424	}
	425
	426	static inline void node_set_state(int node, enum node_states state)
	427	{
	428	__node_set(node, &node_states[state]);
	429	}
	430
	431	static inline void node_clear_state(int node, enum node_states state)
	432	{
	433	__node_clear(node, &node_states[state]);
	434	}
	435
	436	static inline int num_node_state(enum node_states state)
	437	{
	438	return nodes_weight(node_states[state]);
	439	}
	440
	441	#define for_each_node_state(__node, __state) \
	442	for_each_node_mask((__node), node_states[__state])
	443
	444	#define first_online_node first_node(node_states[N_ONLINE])
8d060bf4	445	#define first_memory_node first_node(node_states[N_MEMORY])
0dfe5407	446	static inline unsigned int next_online_node(int nid)
8d060bf4 DR	447	{
	448	return next_node(nid, node_states[N_ONLINE]);
	449	}
0dfe5407	450	static inline unsigned int next_memory_node(int nid)
8d060bf4 DR	451	{
	452	return next_node(nid, node_states[N_MEMORY]);
	453	}
13808910	454
b9726c26	455	extern unsigned int nr_node_ids;
ce0725f7	456	extern unsigned int nr_online_nodes;
62bc62a8 CL	457
	458	static inline void node_set_online(int nid)
	459	{
	460	node_set_state(nid, N_ONLINE);
	461	nr_online_nodes = num_node_state(N_ONLINE);
	462	}
	463
	464	static inline void node_set_offline(int nid)
	465	{
	466	node_clear_state(nid, N_ONLINE);
	467	nr_online_nodes = num_node_state(N_ONLINE);
	468	}
778d3b0f	469
1da177e4	470	#else
13808910 CL	471
	472	static inline int node_state(int node, enum node_states state)
	473	{
	474	return node == 0;
	475	}
	476
	477	static inline void node_set_state(int node, enum node_states state)
	478	{
	479	}
	480
	481	static inline void node_clear_state(int node, enum node_states state)
	482	{
	483	}
	484
	485	static inline int num_node_state(enum node_states state)
	486	{
	487	return 1;
	488	}
	489
	490	#define for_each_node_state(node, __state) \
	491	for ( (node) = 0; (node) == 0; (node) = 1)
	492
8357f869	493	#define first_online_node 0
8d060bf4	494	#define first_memory_node 0
8357f869	495	#define next_online_node(nid) (MAX_NUMNODES)
b9726c26	496	#define nr_node_ids 1U
ce0725f7	497	#define nr_online_nodes 1U
13808910	498
62bc62a8 CL	499	#define node_set_online(node) node_set_state((node), N_ONLINE)
62bc62a8 CL	500	#define node_set_offline(node) node_clear_state((node), N_ONLINE)
778d3b0f MH	501
	502	#endif
	503
36d4b36b YN	504	static inline int node_random(const nodemask_t *maskp)
36d4b36b YN	505	{
778d3b0f	506	#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
36d4b36b YN	507	int w, bit = NUMA_NO_NODE;
	508
	509	w = nodes_weight(*maskp);
	510	if (w)
	511	bit = bitmap_ord_to_pos(maskp->bits,
	512	get_random_int() % w, MAX_NUMNODES);
	513	return bit;
778d3b0f	514	#else
778d3b0f	515	return 0;
1da177e4	516	#endif
36d4b36b	517	}
1da177e4	518
13808910 CL	519	#define node_online_map node_states[N_ONLINE]
	520	#define node_possible_map node_states[N_POSSIBLE]
	521
13808910 CL	522	#define num_online_nodes() num_node_state(N_ONLINE)
	523	#define num_possible_nodes() num_node_state(N_POSSIBLE)
	524	#define node_online(node) node_state((node), N_ONLINE)
	525	#define node_possible(node) node_state((node), N_POSSIBLE)
	526
13808910 CL	527	#define for_each_node(node) for_each_node_state(node, N_POSSIBLE)
13808910 CL	528	#define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
1da177e4	529
4bfc4495	530	/*
c23c8082	531	* For nodemask scratch area.
bad44b5b DR	532	* NODEMASK_ALLOC(type, name) allocates an object with a specified type and
bad44b5b DR	533	* name.
4bfc4495	534	*/
5df66d30	535	#if NODES_SHIFT > 8 /* nodemask_t > 32 bytes */
bad44b5b DR	536	#define NODEMASK_ALLOC(type, name, gfp_flags) \
	537	type name = kmalloc(sizeof(name), gfp_flags)
	538	#define NODEMASK_FREE(m) kfree(m)
4bfc4495	539	#else
7baab93f	540	#define NODEMASK_ALLOC(type, name, gfp_flags) type _##name, *name = &_##name
bad44b5b	541	#define NODEMASK_FREE(m) do {} while (0)
4bfc4495 KH	542	#endif
4bfc4495 KH	543
c23c8082	544	/* Example structure for using NODEMASK_ALLOC, used in mempolicy. */
4bfc4495 KH	545	struct nodemask_scratch {
	546	nodemask_t mask1;
	547	nodemask_t mask2;
	548	};
	549
bad44b5b DR	550	#define NODEMASK_SCRATCH(x) \
	551	NODEMASK_ALLOC(struct nodemask_scratch, x, \
	552	GFP_KERNEL \| __GFP_NORETRY)
4e7b8a6c	553	#define NODEMASK_SCRATCH_FREE(x) NODEMASK_FREE(x)
4bfc4495 KH	554
4bfc4495 KH	555
1da177e4	556	#endif /* __LINUX_NODEMASK_H */