[linux.git] / fs / io-wq.h

#ifndef INTERNAL_IO_WQ_H
#define INTERNAL_IO_WQ_H

#include <linux/refcount.h>
#include <linux/io_uring.h>

struct io_wq;

enum {
	IO_WQ_WORK_CANCEL	= 1,
	IO_WQ_WORK_HASHED	= 2,
	IO_WQ_WORK_UNBOUND	= 4,
	IO_WQ_WORK_CONCURRENT	= 16,

	IO_WQ_HASH_SHIFT	= 24,	/* upper 8 bits are used for hash key */
};

enum io_wq_cancel {
	IO_WQ_CANCEL_OK,	/* cancelled before started */
	IO_WQ_CANCEL_RUNNING,	/* found, running, and attempted cancelled */
	IO_WQ_CANCEL_NOTFOUND,	/* work not found */
};

static inline void wq_list_add_after(struct io_wq_work_node *node,
				     struct io_wq_work_node *pos,
				     struct io_wq_work_list *list)
{
	struct io_wq_work_node *next = pos->next;

	pos->next = node;
	node->next = next;
	if (!next)
		list->last = node;
}

static inline void wq_list_add_tail(struct io_wq_work_node *node,
				    struct io_wq_work_list *list)
{
	if (!list->first) {
		list->last = node;
		WRITE_ONCE(list->first, node);
	} else {
		list->last->next = node;
		list->last = node;
	}
	node->next = NULL;
}

static inline void wq_list_cut(struct io_wq_work_list *list,
			       struct io_wq_work_node *last,
			       struct io_wq_work_node *prev)
{
	/* first in the list, if prev==NULL */
	if (!prev)
		WRITE_ONCE(list->first, last->next);
	else
		prev->next = last->next;

	if (last == list->last)
		list->last = prev;
	last->next = NULL;
}

static inline void wq_list_del(struct io_wq_work_list *list,
			       struct io_wq_work_node *node,
			       struct io_wq_work_node *prev)
{
	wq_list_cut(list, node, prev);
}

#define wq_list_for_each(pos, prv, head)			\
	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)

#define wq_list_empty(list)	(READ_ONCE((list)->first) == NULL)
#define INIT_WQ_LIST(list)	do {				\
	(list)->first = NULL;					\
	(list)->last = NULL;					\
} while (0)

struct io_wq_work {
	struct io_wq_work_node list;
	const struct cred *creds;
	unsigned flags;
};

static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
{
	if (!work->list.next)
		return NULL;

	return container_of(work->list.next, struct io_wq_work, list);
}

typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *);
typedef void (io_wq_work_fn)(struct io_wq_work *);

struct io_wq_hash {
	refcount_t refs;
	unsigned long map;
	struct wait_queue_head wait;
};

static inline void io_wq_put_hash(struct io_wq_hash *hash)
{
	if (refcount_dec_and_test(&hash->refs))
		kfree(hash);
}

struct io_wq_data {
	struct io_wq_hash *hash;
	io_wq_work_fn *do_work;
	free_work_fn *free_work;
};

struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
void io_wq_put(struct io_wq *wq);
void io_wq_put_and_exit(struct io_wq *wq);

void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
void io_wq_hash_work(struct io_wq_work *work, void *val);

static inline bool io_wq_is_hashed(struct io_wq_work *work)
{
	return work->flags & IO_WQ_WORK_HASHED;
}

typedef bool (work_cancel_fn)(struct io_wq_work *, void *);

enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
					void *data, bool cancel_all);

#if defined(CONFIG_IO_WQ)
extern void io_wq_worker_sleeping(struct task_struct *);
extern void io_wq_worker_running(struct task_struct *);
#else
static inline void io_wq_worker_sleeping(struct task_struct *tsk)
{
}
static inline void io_wq_worker_running(struct task_struct *tsk)
{
}
#endif

static inline bool io_wq_current_is_worker(void)
{
	return in_task() && (current->flags & PF_IO_WORKER) &&
		current->pf_io_worker;
}
#endif
Commit	Line	Data
771b53d0 JA	1	#ifndef INTERNAL_IO_WQ_H
	2	#define INTERNAL_IO_WQ_H
	3
e941894e	4	#include <linux/refcount.h>
98447d65 JA	5	#include <linux/io_uring.h>
98447d65 JA	6
771b53d0 JA	7	struct io_wq;
	8
	9	enum {
	10	IO_WQ_WORK_CANCEL = 1,
e883a79d PB	11	IO_WQ_WORK_HASHED = 2,
e883a79d PB	12	IO_WQ_WORK_UNBOUND = 4,
e883a79d	13	IO_WQ_WORK_CONCURRENT = 16,
771b53d0 JA	14
	15	IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */
	16	};
	17
	18	enum io_wq_cancel {
	19	IO_WQ_CANCEL_OK, /* cancelled before started */
	20	IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */
	21	IO_WQ_CANCEL_NOTFOUND, /* work not found */
	22	};
	23
86f3cd1b PB	24	static inline void wq_list_add_after(struct io_wq_work_node *node,
	25	struct io_wq_work_node *pos,
	26	struct io_wq_work_list *list)
	27	{
	28	struct io_wq_work_node *next = pos->next;
	29
	30	pos->next = node;
	31	node->next = next;
	32	if (!next)
	33	list->last = node;
	34	}
	35
6206f0e1 JA	36	static inline void wq_list_add_tail(struct io_wq_work_node *node,
	37	struct io_wq_work_list *list)
	38	{
	39	if (!list->first) {
e995d512 JA	40	list->last = node;
e995d512 JA	41	WRITE_ONCE(list->first, node);
6206f0e1 JA	42	} else {
	43	list->last->next = node;
	44	list->last = node;
	45	}
0020ef04	46	node->next = NULL;
6206f0e1 JA	47	}
6206f0e1 JA	48
86f3cd1b PB	49	static inline void wq_list_cut(struct io_wq_work_list *list,
86f3cd1b PB	50	struct io_wq_work_node *last,
6206f0e1 JA	51	struct io_wq_work_node *prev)
6206f0e1 JA	52	{
86f3cd1b PB	53	/* first in the list, if prev==NULL */
	54	if (!prev)
	55	WRITE_ONCE(list->first, last->next);
	56	else
	57	prev->next = last->next;
	58
	59	if (last == list->last)
6206f0e1	60	list->last = prev;
86f3cd1b PB	61	last->next = NULL;
	62	}
	63
	64	static inline void wq_list_del(struct io_wq_work_list *list,
	65	struct io_wq_work_node *node,
	66	struct io_wq_work_node *prev)
	67	{
	68	wq_list_cut(list, node, prev);
6206f0e1 JA	69	}
	70
	71	#define wq_list_for_each(pos, prv, head) \
	72	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
	73
e995d512	74	#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL)
6206f0e1 JA	75	#define INIT_WQ_LIST(list) do { \
	76	(list)->first = NULL; \
	77	(list)->last = NULL; \
	78	} while (0)
	79
771b53d0	80	struct io_wq_work {
18a542ff	81	struct io_wq_work_node list;
003e8dcc	82	const struct cred *creds;
6206f0e1	83	unsigned flags;
771b53d0 JA	84	};
771b53d0 JA	85
86f3cd1b PB	86	static inline struct io_wq_work wq_next_work(struct io_wq_work work)
	87	{
	88	if (!work->list.next)
	89	return NULL;
	90
	91	return container_of(work->list.next, struct io_wq_work, list);
	92	}
	93
5280f7e5 PB	94	typedef struct io_wq_work (free_work_fn)(struct io_wq_work );
5280f7e5 PB	95	typedef void (io_wq_work_fn)(struct io_wq_work *);
7d723065	96
e941894e JA	97	struct io_wq_hash {
	98	refcount_t refs;
	99	unsigned long map;
	100	struct wait_queue_head wait;
	101	};
	102
	103	static inline void io_wq_put_hash(struct io_wq_hash *hash)
	104	{
	105	if (refcount_dec_and_test(&hash->refs))
	106	kfree(hash);
	107	}
	108
576a347b	109	struct io_wq_data {
e941894e	110	struct io_wq_hash *hash;
f5fa38c5	111	io_wq_work_fn *do_work;
e9fd9396	112	free_work_fn *free_work;
576a347b JA	113	};
	114
	115	struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data);
4fb6ac32	116	void io_wq_put(struct io_wq *wq);
afcc4015	117	void io_wq_put_and_exit(struct io_wq *wq);
771b53d0 JA	118
771b53d0 JA	119	void io_wq_enqueue(struct io_wq wq, struct io_wq_work work);
8766dd51 PB	120	void io_wq_hash_work(struct io_wq_work work, void val);
	121
	122	static inline bool io_wq_is_hashed(struct io_wq_work *work)
	123	{
	124	return work->flags & IO_WQ_WORK_HASHED;
	125	}
771b53d0	126
62755e35 JA	127	typedef bool (work_cancel_fn)(struct io_wq_work , void );
	128
	129	enum io_wq_cancel io_wq_cancel_cb(struct io_wq wq, work_cancel_fn cancel,
4f26bda1	130	void *data, bool cancel_all);
62755e35	131
771b53d0 JA	132	#if defined(CONFIG_IO_WQ)
	133	extern void io_wq_worker_sleeping(struct task_struct *);
	134	extern void io_wq_worker_running(struct task_struct *);
	135	#else
	136	static inline void io_wq_worker_sleeping(struct task_struct *tsk)
	137	{
	138	}
	139	static inline void io_wq_worker_running(struct task_struct *tsk)
	140	{
	141	}
525b305d	142	#endif
771b53d0	143
525b305d JA	144	static inline bool io_wq_current_is_worker(void)
525b305d JA	145	{
3bfe6106 JA	146	return in_task() && (current->flags & PF_IO_WORKER) &&
3bfe6106 JA	147	current->pf_io_worker;
525b305d JA	148	}
525b305d JA	149	#endif