[linux.git] / lib / ref_tracker.c

// SPDX-License-Identifier: GPL-2.0-or-later

#define pr_fmt(fmt) "ref_tracker: " fmt

#include <linux/export.h>
#include <linux/list_sort.h>
#include <linux/ref_tracker.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/stackdepot.h>

#define REF_TRACKER_STACK_ENTRIES 16
#define STACK_BUF_SIZE 1024

struct ref_tracker {
	struct list_head	head;   /* anchor into dir->list or dir->quarantine */
	bool			dead;
	depot_stack_handle_t	alloc_stack_handle;
	depot_stack_handle_t	free_stack_handle;
};

struct ref_tracker_dir_stats {
	int total;
	int count;
	struct {
		depot_stack_handle_t stack_handle;
		unsigned int count;
	} stacks[];
};

static struct ref_tracker_dir_stats *
ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
{
	struct ref_tracker_dir_stats *stats;
	struct ref_tracker *tracker;

	stats = kmalloc(struct_size(stats, stacks, limit),
			GFP_NOWAIT | __GFP_NOWARN);
	if (!stats)
		return ERR_PTR(-ENOMEM);
	stats->total = 0;
	stats->count = 0;

	list_for_each_entry(tracker, &dir->list, head) {
		depot_stack_handle_t stack = tracker->alloc_stack_handle;
		int i;

		++stats->total;
		for (i = 0; i < stats->count; ++i)
			if (stats->stacks[i].stack_handle == stack)
				break;
		if (i >= limit)
			continue;
		if (i >= stats->count) {
			stats->stacks[i].stack_handle = stack;
			stats->stacks[i].count = 0;
			++stats->count;
		}
		++stats->stacks[i].count;
	}

	return stats;
}

struct ostream {
	char *buf;
	int size, used;
};

#define pr_ostream(stream, fmt, args...) \
({ \
	struct ostream *_s = (stream); \
\
	if (!_s->buf) { \
		pr_err(fmt, ##args); \
	} else { \
		int ret, len = _s->size - _s->used; \
		ret = snprintf(_s->buf + _s->used, len, pr_fmt(fmt), ##args); \
		_s->used += min(ret, len); \
	} \
})

static void
__ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir,
			     unsigned int display_limit, struct ostream *s)
{
	struct ref_tracker_dir_stats *stats;
	unsigned int i = 0, skipped;
	depot_stack_handle_t stack;
	char *sbuf;

	lockdep_assert_held(&dir->lock);

	if (list_empty(&dir->list))
		return;

	stats = ref_tracker_get_stats(dir, display_limit);
	if (IS_ERR(stats)) {
		pr_ostream(s, "%s@%pK: couldn't get stats, error %pe\n",
			   dir->name, dir, stats);
		return;
	}

	sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT | __GFP_NOWARN);

	for (i = 0, skipped = stats->total; i < stats->count; ++i) {
		stack = stats->stacks[i].stack_handle;
		if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4))
			sbuf[0] = 0;
		pr_ostream(s, "%s@%pK has %d/%d users at\n%s\n", dir->name, dir,
			   stats->stacks[i].count, stats->total, sbuf);
		skipped -= stats->stacks[i].count;
	}

	if (skipped)
		pr_ostream(s, "%s@%pK skipped reports about %d/%d users.\n",
			   dir->name, dir, skipped, stats->total);

	kfree(sbuf);

	kfree(stats);
}

void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir,
				  unsigned int display_limit)
{
	struct ostream os = {};

	__ref_tracker_dir_pr_ostream(dir, display_limit, &os);
}
EXPORT_SYMBOL(ref_tracker_dir_print_locked);

void ref_tracker_dir_print(struct ref_tracker_dir *dir,
			   unsigned int display_limit)
{
	unsigned long flags;

	spin_lock_irqsave(&dir->lock, flags);
	ref_tracker_dir_print_locked(dir, display_limit);
	spin_unlock_irqrestore(&dir->lock, flags);
}
EXPORT_SYMBOL(ref_tracker_dir_print);

int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size)
{
	struct ostream os = { .buf = buf, .size = size };
	unsigned long flags;

	spin_lock_irqsave(&dir->lock, flags);
	__ref_tracker_dir_pr_ostream(dir, 16, &os);
	spin_unlock_irqrestore(&dir->lock, flags);

	return os.used;
}
EXPORT_SYMBOL(ref_tracker_dir_snprint);

void ref_tracker_dir_exit(struct ref_tracker_dir *dir)
{
	struct ref_tracker *tracker, *n;
	unsigned long flags;
	bool leak = false;

	dir->dead = true;
	spin_lock_irqsave(&dir->lock, flags);
	list_for_each_entry_safe(tracker, n, &dir->quarantine, head) {
		list_del(&tracker->head);
		kfree(tracker);
		dir->quarantine_avail++;
	}
	if (!list_empty(&dir->list)) {
		ref_tracker_dir_print_locked(dir, 16);
		leak = true;
		list_for_each_entry_safe(tracker, n, &dir->list, head) {
			list_del(&tracker->head);
			kfree(tracker);
		}
	}
	spin_unlock_irqrestore(&dir->lock, flags);
	WARN_ON_ONCE(leak);
	WARN_ON_ONCE(refcount_read(&dir->untracked) != 1);
	WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1);
}
EXPORT_SYMBOL(ref_tracker_dir_exit);

int ref_tracker_alloc(struct ref_tracker_dir *dir,
		      struct ref_tracker **trackerp,
		      gfp_t gfp)
{
	unsigned long entries[REF_TRACKER_STACK_ENTRIES];
	struct ref_tracker *tracker;
	unsigned int nr_entries;
	gfp_t gfp_mask = gfp | __GFP_NOWARN;
	unsigned long flags;

	WARN_ON_ONCE(dir->dead);

	if (!trackerp) {
		refcount_inc(&dir->no_tracker);
		return 0;
	}
	if (gfp & __GFP_DIRECT_RECLAIM)
		gfp_mask |= __GFP_NOFAIL;
	*trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask);
	if (unlikely(!tracker)) {
		pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
		refcount_inc(&dir->untracked);
		return -ENOMEM;
	}
	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
	tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp);

	spin_lock_irqsave(&dir->lock, flags);
	list_add(&tracker->head, &dir->list);
	spin_unlock_irqrestore(&dir->lock, flags);
	return 0;
}
EXPORT_SYMBOL_GPL(ref_tracker_alloc);

int ref_tracker_free(struct ref_tracker_dir *dir,
		     struct ref_tracker **trackerp)
{
	unsigned long entries[REF_TRACKER_STACK_ENTRIES];
	depot_stack_handle_t stack_handle;
	struct ref_tracker *tracker;
	unsigned int nr_entries;
	unsigned long flags;

	WARN_ON_ONCE(dir->dead);

	if (!trackerp) {
		refcount_dec(&dir->no_tracker);
		return 0;
	}
	tracker = *trackerp;
	if (!tracker) {
		refcount_dec(&dir->untracked);
		return -EEXIST;
	}
	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
	stack_handle = stack_depot_save(entries, nr_entries,
					GFP_NOWAIT | __GFP_NOWARN);

	spin_lock_irqsave(&dir->lock, flags);
	if (tracker->dead) {
		pr_err("reference already released.\n");
		if (tracker->alloc_stack_handle) {
			pr_err("allocated in:\n");
			stack_depot_print(tracker->alloc_stack_handle);
		}
		if (tracker->free_stack_handle) {
			pr_err("freed in:\n");
			stack_depot_print(tracker->free_stack_handle);
		}
		spin_unlock_irqrestore(&dir->lock, flags);
		WARN_ON_ONCE(1);
		return -EINVAL;
	}
	tracker->dead = true;

	tracker->free_stack_handle = stack_handle;

	list_move_tail(&tracker->head, &dir->quarantine);
	if (!dir->quarantine_avail) {
		tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head);
		list_del(&tracker->head);
	} else {
		dir->quarantine_avail--;
		tracker = NULL;
	}
	spin_unlock_irqrestore(&dir->lock, flags);

	kfree(tracker);
	return 0;
}
EXPORT_SYMBOL_GPL(ref_tracker_free);
Commit	Line	Data
4e66934e	1	// SPDX-License-Identifier: GPL-2.0-or-later
b6d7c0eb AH	2
	3	#define pr_fmt(fmt) "ref_tracker: " fmt
	4
4e66934e	5	#include <linux/export.h>
b6d7c0eb	6	#include <linux/list_sort.h>
4e66934e ED	7	#include <linux/ref_tracker.h>
	8	#include <linux/slab.h>
	9	#include <linux/stacktrace.h>
	10	#include <linux/stackdepot.h>
	11
	12	#define REF_TRACKER_STACK_ENTRIES 16
b6d7c0eb	13	#define STACK_BUF_SIZE 1024
4e66934e ED	14
	15	struct ref_tracker {
	16	struct list_head head; /* anchor into dir->list or dir->quarantine */
	17	bool dead;
	18	depot_stack_handle_t alloc_stack_handle;
	19	depot_stack_handle_t free_stack_handle;
	20	};
	21
b6d7c0eb AH	22	struct ref_tracker_dir_stats {
	23	int total;
	24	int count;
	25	struct {
	26	depot_stack_handle_t stack_handle;
	27	unsigned int count;
	28	} stacks[];
	29	};
	30
	31	static struct ref_tracker_dir_stats *
	32	ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
7a113ff6	33	{
b6d7c0eb	34	struct ref_tracker_dir_stats *stats;
7a113ff6	35	struct ref_tracker *tracker;
7a113ff6	36
b6d7c0eb AH	37	stats = kmalloc(struct_size(stats, stacks, limit),
	38	GFP_NOWAIT \| __GFP_NOWARN);
	39	if (!stats)
	40	return ERR_PTR(-ENOMEM);
	41	stats->total = 0;
	42	stats->count = 0;
7a113ff6 AH	43
7a113ff6 AH	44	list_for_each_entry(tracker, &dir->list, head) {
b6d7c0eb AH	45	depot_stack_handle_t stack = tracker->alloc_stack_handle;
	46	int i;
	47
	48	++stats->total;
	49	for (i = 0; i < stats->count; ++i)
	50	if (stats->stacks[i].stack_handle == stack)
	51	break;
	52	if (i >= limit)
	53	continue;
	54	if (i >= stats->count) {
	55	stats->stacks[i].stack_handle = stack;
	56	stats->stacks[i].count = 0;
	57	++stats->count;
7a113ff6	58	}
b6d7c0eb	59	++stats->stacks[i].count;
7a113ff6	60	}
b6d7c0eb AH	61
	62	return stats;
	63	}
	64
227c6c83 AH	65	struct ostream {
	66	char *buf;
	67	int size, used;
	68	};
	69
	70	#define pr_ostream(stream, fmt, args...) \
	71	({ \
	72	struct ostream *_s = (stream); \
	73	\
	74	if (!_s->buf) { \
	75	pr_err(fmt, ##args); \
	76	} else { \
	77	int ret, len = _s->size - _s->used; \
	78	ret = snprintf(_s->buf + _s->used, len, pr_fmt(fmt), ##args); \
	79	_s->used += min(ret, len); \
	80	} \
	81	})
	82
	83	static void
	84	__ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir,
	85	unsigned int display_limit, struct ostream *s)
b6d7c0eb AH	86	{
	87	struct ref_tracker_dir_stats *stats;
	88	unsigned int i = 0, skipped;
	89	depot_stack_handle_t stack;
	90	char *sbuf;
	91
	92	lockdep_assert_held(&dir->lock);
	93
	94	if (list_empty(&dir->list))
	95	return;
	96
	97	stats = ref_tracker_get_stats(dir, display_limit);
	98	if (IS_ERR(stats)) {
227c6c83 AH	99	pr_ostream(s, "%s@%pK: couldn't get stats, error %pe\n",
227c6c83 AH	100	dir->name, dir, stats);
b6d7c0eb AH	101	return;
	102	}
	103
	104	sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT \| __GFP_NOWARN);
	105
	106	for (i = 0, skipped = stats->total; i < stats->count; ++i) {
	107	stack = stats->stacks[i].stack_handle;
	108	if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4))
	109	sbuf[0] = 0;
227c6c83 AH	110	pr_ostream(s, "%s@%pK has %d/%d users at\n%s\n", dir->name, dir,
227c6c83 AH	111	stats->stacks[i].count, stats->total, sbuf);
b6d7c0eb AH	112	skipped -= stats->stacks[i].count;
	113	}
	114
	115	if (skipped)
227c6c83 AH	116	pr_ostream(s, "%s@%pK skipped reports about %d/%d users.\n",
227c6c83 AH	117	dir->name, dir, skipped, stats->total);
b6d7c0eb AH	118
	119	kfree(sbuf);
	120
	121	kfree(stats);
7a113ff6	122	}
227c6c83 AH	123
	124	void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir,
	125	unsigned int display_limit)
	126	{
	127	struct ostream os = {};
	128
	129	__ref_tracker_dir_pr_ostream(dir, display_limit, &os);
	130	}
7a113ff6 AH	131	EXPORT_SYMBOL(ref_tracker_dir_print_locked);
	132
	133	void ref_tracker_dir_print(struct ref_tracker_dir *dir,
	134	unsigned int display_limit)
	135	{
	136	unsigned long flags;
	137
	138	spin_lock_irqsave(&dir->lock, flags);
	139	ref_tracker_dir_print_locked(dir, display_limit);
	140	spin_unlock_irqrestore(&dir->lock, flags);
	141	}
	142	EXPORT_SYMBOL(ref_tracker_dir_print);
	143
227c6c83 AH	144	int ref_tracker_dir_snprint(struct ref_tracker_dir dir, char buf, size_t size)
	145	{
	146	struct ostream os = { .buf = buf, .size = size };
	147	unsigned long flags;
	148
	149	spin_lock_irqsave(&dir->lock, flags);
	150	__ref_tracker_dir_pr_ostream(dir, 16, &os);
	151	spin_unlock_irqrestore(&dir->lock, flags);
	152
	153	return os.used;
	154	}
	155	EXPORT_SYMBOL(ref_tracker_dir_snprint);
	156
4e66934e ED	157	void ref_tracker_dir_exit(struct ref_tracker_dir *dir)
	158	{
	159	struct ref_tracker tracker, n;
	160	unsigned long flags;
	161	bool leak = false;
	162
e3ececfe	163	dir->dead = true;
4e66934e ED	164	spin_lock_irqsave(&dir->lock, flags);
	165	list_for_each_entry_safe(tracker, n, &dir->quarantine, head) {
	166	list_del(&tracker->head);
	167	kfree(tracker);
	168	dir->quarantine_avail++;
	169	}
7a113ff6 AH	170	if (!list_empty(&dir->list)) {
7a113ff6 AH	171	ref_tracker_dir_print_locked(dir, 16);
4e66934e	172	leak = true;
7a113ff6 AH	173	list_for_each_entry_safe(tracker, n, &dir->list, head) {
	174	list_del(&tracker->head);
	175	kfree(tracker);
	176	}
4e66934e ED	177	}
	178	spin_unlock_irqrestore(&dir->lock, flags);
	179	WARN_ON_ONCE(leak);
	180	WARN_ON_ONCE(refcount_read(&dir->untracked) != 1);
8fd5522f	181	WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1);
4e66934e ED	182	}
	183	EXPORT_SYMBOL(ref_tracker_dir_exit);
	184
4e66934e ED	185	int ref_tracker_alloc(struct ref_tracker_dir *dir,
	186	struct ref_tracker **trackerp,
	187	gfp_t gfp)
	188	{
	189	unsigned long entries[REF_TRACKER_STACK_ENTRIES];
	190	struct ref_tracker *tracker;
	191	unsigned int nr_entries;
acd8f0e5	192	gfp_t gfp_mask = gfp \| __GFP_NOWARN;
4e66934e ED	193	unsigned long flags;
4e66934e ED	194
e3ececfe ED	195	WARN_ON_ONCE(dir->dead);
e3ececfe ED	196
8fd5522f ED	197	if (!trackerp) {
	198	refcount_inc(&dir->no_tracker);
	199	return 0;
	200	}
c12837d1 ED	201	if (gfp & __GFP_DIRECT_RECLAIM)
	202	gfp_mask \|= __GFP_NOFAIL;
	203	trackerp = tracker = kzalloc(sizeof(tracker), gfp_mask);
4e66934e ED	204	if (unlikely(!tracker)) {
	205	pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
	206	refcount_inc(&dir->untracked);
	207	return -ENOMEM;
	208	}
	209	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
4e66934e ED	210	tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp);
	211
	212	spin_lock_irqsave(&dir->lock, flags);
	213	list_add(&tracker->head, &dir->list);
	214	spin_unlock_irqrestore(&dir->lock, flags);
	215	return 0;
	216	}
	217	EXPORT_SYMBOL_GPL(ref_tracker_alloc);
	218
	219	int ref_tracker_free(struct ref_tracker_dir *dir,
	220	struct ref_tracker **trackerp)
	221	{
	222	unsigned long entries[REF_TRACKER_STACK_ENTRIES];
4e66934e	223	depot_stack_handle_t stack_handle;
8fd5522f	224	struct ref_tracker *tracker;
4e66934e ED	225	unsigned int nr_entries;
	226	unsigned long flags;
	227
e3ececfe ED	228	WARN_ON_ONCE(dir->dead);
e3ececfe ED	229
8fd5522f ED	230	if (!trackerp) {
	231	refcount_dec(&dir->no_tracker);
	232	return 0;
	233	}
	234	tracker = *trackerp;
4e66934e ED	235	if (!tracker) {
	236	refcount_dec(&dir->untracked);
	237	return -EEXIST;
	238	}
	239	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
acd8f0e5 AH	240	stack_handle = stack_depot_save(entries, nr_entries,
acd8f0e5 AH	241	GFP_NOWAIT \| __GFP_NOWARN);
4e66934e ED	242
	243	spin_lock_irqsave(&dir->lock, flags);
	244	if (tracker->dead) {
	245	pr_err("reference already released.\n");
	246	if (tracker->alloc_stack_handle) {
	247	pr_err("allocated in:\n");
	248	stack_depot_print(tracker->alloc_stack_handle);
	249	}
	250	if (tracker->free_stack_handle) {
	251	pr_err("freed in:\n");
	252	stack_depot_print(tracker->free_stack_handle);
	253	}
	254	spin_unlock_irqrestore(&dir->lock, flags);
	255	WARN_ON_ONCE(1);
	256	return -EINVAL;
	257	}
	258	tracker->dead = true;
	259
	260	tracker->free_stack_handle = stack_handle;
	261
	262	list_move_tail(&tracker->head, &dir->quarantine);
	263	if (!dir->quarantine_avail) {
	264	tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head);
	265	list_del(&tracker->head);
	266	} else {
	267	dir->quarantine_avail--;
	268	tracker = NULL;
	269	}
	270	spin_unlock_irqrestore(&dir->lock, flags);
	271
	272	kfree(tracker);
	273	return 0;
	274	}
	275	EXPORT_SYMBOL_GPL(ref_tracker_free);