Git Repo - linux.git/blame - block/blk-cgroup-rwstat.h

Commit	Line	Data
1d156646 TH	1	/* SPDX-License-Identifier: GPL-2.0
	2	*
	3	* Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
	4	* Do not use in new code.
	5	*/
	6	#ifndef _BLK_CGROUP_RWSTAT_H
	7	#define _BLK_CGROUP_RWSTAT_H
	8
672fdcf0	9	#include "blk-cgroup.h"
1d156646 TH	10
	11	enum blkg_rwstat_type {
	12	BLKG_RWSTAT_READ,
	13	BLKG_RWSTAT_WRITE,
	14	BLKG_RWSTAT_SYNC,
	15	BLKG_RWSTAT_ASYNC,
	16	BLKG_RWSTAT_DISCARD,
	17
	18	BLKG_RWSTAT_NR,
	19	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
	20	};
	21
	22	/*
	23	* blkg_[rw]stat->aux_cnt is excluded for local stats but included for
	24	* recursive. Used to carry stats of dead children.
	25	*/
	26	struct blkg_rwstat {
	27	struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
	28	atomic64_t aux_cnt[BLKG_RWSTAT_NR];
	29	};
	30
	31	struct blkg_rwstat_sample {
	32	u64 cnt[BLKG_RWSTAT_NR];
	33	};
	34
	35	static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
	36	unsigned int idx)
	37	{
	38	return atomic64_read(&rwstat->aux_cnt[idx]) +
	39	percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
	40	}
	41
	42	int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
	43	void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
	44	u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	45	const struct blkg_rwstat_sample *rwstat);
	46	u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	47	int off);
	48	void blkg_rwstat_recursive_sum(struct blkcg_gq blkg, struct blkcg_policy pol,
	49	int off, struct blkg_rwstat_sample *sum);
	50
	51
	52	/**
	53	* blkg_rwstat_add - add a value to a blkg_rwstat
	54	* @rwstat: target blkg_rwstat
f403034e	55	* @opf: REQ_OP and flags
1d156646 TH	56	* @val: value to add
	57	*
	58	* Add @val to @rwstat. The counters are chosen according to @rw. The
	59	* caller is responsible for synchronizing calls to this function.
	60	*/
	61	static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
16458cf3	62	blk_opf_t opf, uint64_t val)
1d156646 TH	63	{
	64	struct percpu_counter *cnt;
	65
16458cf3	66	if (op_is_discard(opf))
1d156646	67	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
16458cf3	68	else if (op_is_write(opf))
1d156646 TH	69	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
	70	else
	71	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
	72
	73	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
	74
16458cf3	75	if (op_is_sync(opf))
1d156646 TH	76	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
	77	else
	78	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
	79
	80	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
	81	}
	82
	83	/**
	84	* blkg_rwstat_read - read the current values of a blkg_rwstat
	85	* @rwstat: blkg_rwstat to read
f403034e	86	* @result: where to put the current values
1d156646	87	*
f403034e	88	* Read the current snapshot of @rwstat and return it in the @result counts.
1d156646 TH	89	*/
	90	static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
	91	struct blkg_rwstat_sample *result)
	92	{
	93	int i;
	94
	95	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	96	result->cnt[i] =
	97	percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
	98	}
	99
	100	/**
	101	* blkg_rwstat_total - read the total count of a blkg_rwstat
	102	* @rwstat: blkg_rwstat to read
	103	*
	104	* Return the total count of @rwstat regardless of the IO direction. This
	105	* function can be called without synchronization and takes care of u64
	106	* atomicity.
	107	*/
	108	static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
	109	{
	110	struct blkg_rwstat_sample tmp = { };
	111
	112	blkg_rwstat_read(rwstat, &tmp);
	113	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
	114	}
	115
	116	/**
	117	* blkg_rwstat_reset - reset a blkg_rwstat
	118	* @rwstat: blkg_rwstat to reset
	119	*/
	120	static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
	121	{
	122	int i;
	123
	124	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
	125	percpu_counter_set(&rwstat->cpu_cnt[i], 0);
	126	atomic64_set(&rwstat->aux_cnt[i], 0);
	127	}
	128	}
	129
	130	/**
	131	* blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
	132	* @to: the destination blkg_rwstat
	133	* @from: the source
	134	*
	135	* Add @from's count including the aux one to @to's aux count.
	136	*/
	137	static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
	138	struct blkg_rwstat *from)
	139	{
	140	u64 sum[BLKG_RWSTAT_NR];
	141	int i;
	142
	143	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	144	sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
	145
	146	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	147	atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
	148	&to->aux_cnt[i]);
	149	}
	150	#endif /* _BLK_CGROUP_RWSTAT_H */