[linux.git] / kernel / trace / trace_probe_tmpl.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Traceprobe fetch helper inlines
 */

static nokprobe_inline void
fetch_store_raw(unsigned long val, struct fetch_insn *code, void *buf)
{
	switch (code->size) {
	case 1:
		*(u8 *)buf = (u8)val;
		break;
	case 2:
		*(u16 *)buf = (u16)val;
		break;
	case 4:
		*(u32 *)buf = (u32)val;
		break;
	case 8:
		//TBD: 32bit signed
		*(u64 *)buf = (u64)val;
		break;
	default:
		*(unsigned long *)buf = val;
	}
}

static nokprobe_inline void
fetch_apply_bitfield(struct fetch_insn *code, void *buf)
{
	switch (code->basesize) {
	case 1:
		*(u8 *)buf <<= code->lshift;
		*(u8 *)buf >>= code->rshift;
		break;
	case 2:
		*(u16 *)buf <<= code->lshift;
		*(u16 *)buf >>= code->rshift;
		break;
	case 4:
		*(u32 *)buf <<= code->lshift;
		*(u32 *)buf >>= code->rshift;
		break;
	case 8:
		*(u64 *)buf <<= code->lshift;
		*(u64 *)buf >>= code->rshift;
		break;
	}
}

/*
 * These functions must be defined for each callsite.
 * Return consumed dynamic data size (>= 0), or error (< 0).
 * If dest is NULL, don't store result and return required dynamic data size.
 */
static int
process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs,
		   void *dest, void *base);
static nokprobe_inline int fetch_store_strlen(unsigned long addr);
static nokprobe_inline int
fetch_store_string(unsigned long addr, void *dest, void *base);
static nokprobe_inline int fetch_store_strlen_user(unsigned long addr);
static nokprobe_inline int
fetch_store_string_user(unsigned long addr, void *dest, void *base);
static nokprobe_inline int
probe_mem_read(void *dest, void *src, size_t size);
static nokprobe_inline int
probe_mem_read_user(void *dest, void *src, size_t size);

/* From the 2nd stage, routine is same */
static nokprobe_inline int
process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
			   void *dest, void *base)
{
	struct fetch_insn *s3 = NULL;
	int total = 0, ret = 0, i = 0;
	u32 loc = 0;
	unsigned long lval = val;

stage2:
	/* 2nd stage: dereference memory if needed */
	do {
		if (code->op == FETCH_OP_DEREF) {
			lval = val;
			ret = probe_mem_read(&val, (void *)val + code->offset,
					     sizeof(val));
		} else if (code->op == FETCH_OP_UDEREF) {
			lval = val;
			ret = probe_mem_read_user(&val,
				 (void *)val + code->offset, sizeof(val));
		} else
			break;
		if (ret)
			return ret;
		code++;
	} while (1);

	s3 = code;
stage3:
	/* 3rd stage: store value to buffer */
	if (unlikely(!dest)) {
		if (code->op == FETCH_OP_ST_STRING) {
			ret = fetch_store_strlen(val + code->offset);
			code++;
			goto array;
		} else if (code->op == FETCH_OP_ST_USTRING) {
			ret += fetch_store_strlen_user(val + code->offset);
			code++;
			goto array;
		} else
			return -EILSEQ;
	}

	switch (code->op) {
	case FETCH_OP_ST_RAW:
		fetch_store_raw(val, code, dest);
		break;
	case FETCH_OP_ST_MEM:
		probe_mem_read(dest, (void *)val + code->offset, code->size);
		break;
	case FETCH_OP_ST_UMEM:
		probe_mem_read_user(dest, (void *)val + code->offset, code->size);
		break;
	case FETCH_OP_ST_STRING:
		loc = *(u32 *)dest;
		ret = fetch_store_string(val + code->offset, dest, base);
		break;
	case FETCH_OP_ST_USTRING:
		loc = *(u32 *)dest;
		ret = fetch_store_string_user(val + code->offset, dest, base);
		break;
	default:
		return -EILSEQ;
	}
	code++;

	/* 4th stage: modify stored value if needed */
	if (code->op == FETCH_OP_MOD_BF) {
		fetch_apply_bitfield(code, dest);
		code++;
	}

array:
	/* the last stage: Loop on array */
	if (code->op == FETCH_OP_LP_ARRAY) {
		total += ret;
		if (++i < code->param) {
			code = s3;
			if (s3->op != FETCH_OP_ST_STRING &&
			    s3->op != FETCH_OP_ST_USTRING) {
				dest += s3->size;
				val += s3->size;
				goto stage3;
			}
			code--;
			val = lval + sizeof(char *);
			if (dest) {
				dest += sizeof(u32);
				*(u32 *)dest = update_data_loc(loc, ret);
			}
			goto stage2;
		}
		code++;
		ret = total;
	}

	return code->op == FETCH_OP_END ? ret : -EILSEQ;
}

/* Sum up total data length for dynamic arraies (strings) */
static nokprobe_inline int
__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
{
	struct probe_arg *arg;
	int i, len, ret = 0;

	for (i = 0; i < tp->nr_args; i++) {
		arg = tp->args + i;
		if (unlikely(arg->dynamic)) {
			len = process_fetch_insn(arg->code, regs, NULL, NULL);
			if (len > 0)
				ret += len;
		}
	}

	return ret;
}

/* Store the value of each argument */
static nokprobe_inline void
store_trace_args(void *data, struct trace_probe *tp, struct pt_regs *regs,
		 int header_size, int maxlen)
{
	struct probe_arg *arg;
	void *base = data - header_size;
	void *dyndata = data + tp->size;
	u32 *dl;	/* Data location */
	int ret, i;

	for (i = 0; i < tp->nr_args; i++) {
		arg = tp->args + i;
		dl = data + arg->offset;
		/* Point the dynamic data area if needed */
		if (unlikely(arg->dynamic))
			*dl = make_data_loc(maxlen, dyndata - base);
		ret = process_fetch_insn(arg->code, regs, dl, base);
		if (unlikely(ret < 0 && arg->dynamic)) {
			*dl = make_data_loc(0, dyndata - base);
		} else {
			dyndata += ret;
			maxlen -= ret;
		}
	}
}

static inline int
print_probe_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
		 u8 *data, void *field)
{
	void *p;
	int i, j;

	for (i = 0; i < nr_args; i++) {
		struct probe_arg *a = args + i;

		trace_seq_printf(s, " %s=", a->name);
		if (likely(!a->count)) {
			if (!a->type->print(s, data + a->offset, field))
				return -ENOMEM;
			continue;
		}
		trace_seq_putc(s, '{');
		p = data + a->offset;
		for (j = 0; j < a->count; j++) {
			if (!a->type->print(s, p, field))
				return -ENOMEM;
			trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
			p += a->type->size;
		}
	}
	return 0;
}
Commit	Line	Data
53305928 MH	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Traceprobe fetch helper inlines
	4	*/
	5
	6	static nokprobe_inline void
	7	fetch_store_raw(unsigned long val, struct fetch_insn code, void buf)
	8	{
	9	switch (code->size) {
	10	case 1:
	11	(u8 )buf = (u8)val;
	12	break;
	13	case 2:
	14	(u16 )buf = (u16)val;
	15	break;
	16	case 4:
	17	(u32 )buf = (u32)val;
	18	break;
	19	case 8:
	20	//TBD: 32bit signed
	21	(u64 )buf = (u64)val;
	22	break;
	23	default:
	24	(unsigned long )buf = val;
	25	}
	26	}
	27
	28	static nokprobe_inline void
	29	fetch_apply_bitfield(struct fetch_insn code, void buf)
	30	{
	31	switch (code->basesize) {
	32	case 1:
	33	(u8 )buf <<= code->lshift;
	34	(u8 )buf >>= code->rshift;
	35	break;
	36	case 2:
	37	(u16 )buf <<= code->lshift;
	38	(u16 )buf >>= code->rshift;
	39	break;
	40	case 4:
	41	(u32 )buf <<= code->lshift;
	42	(u32 )buf >>= code->rshift;
	43	break;
	44	case 8:
	45	(u64 )buf <<= code->lshift;
	46	(u64 )buf >>= code->rshift;
	47	break;
	48	}
	49	}
	50
9178412d	51	/*
9b960a38	52	* These functions must be defined for each callsite.
9178412d MH	53	* Return consumed dynamic data size (>= 0), or error (< 0).
	54	* If dest is NULL, don't store result and return required dynamic data size.
	55	*/
53305928 MH	56	static int
53305928 MH	57	process_fetch_insn(struct fetch_insn code, struct pt_regs regs,
9178412d	58	void dest, void base);
9b960a38 MH	59	static nokprobe_inline int fetch_store_strlen(unsigned long addr);
	60	static nokprobe_inline int
	61	fetch_store_string(unsigned long addr, void dest, void base);
88903c46 MH	62	static nokprobe_inline int fetch_store_strlen_user(unsigned long addr);
	63	static nokprobe_inline int
	64	fetch_store_string_user(unsigned long addr, void dest, void base);
9b960a38 MH	65	static nokprobe_inline int
9b960a38 MH	66	probe_mem_read(void dest, void src, size_t size);
e65f7ae7 MH	67	static nokprobe_inline int
e65f7ae7 MH	68	probe_mem_read_user(void dest, void src, size_t size);
9b960a38 MH	69
	70	/* From the 2nd stage, routine is same */
	71	static nokprobe_inline int
	72	process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
	73	void dest, void base)
	74	{
40b53b77 MH	75	struct fetch_insn *s3 = NULL;
	76	int total = 0, ret = 0, i = 0;
	77	u32 loc = 0;
	78	unsigned long lval = val;
9b960a38	79
40b53b77	80	stage2:
9b960a38	81	/* 2nd stage: dereference memory if needed */
e65f7ae7 MH	82	do {
	83	if (code->op == FETCH_OP_DEREF) {
	84	lval = val;
	85	ret = probe_mem_read(&val, (void *)val + code->offset,
	86	sizeof(val));
	87	} else if (code->op == FETCH_OP_UDEREF) {
	88	lval = val;
	89	ret = probe_mem_read_user(&val,
	90	(void *)val + code->offset, sizeof(val));
	91	} else
	92	break;
9b960a38 MH	93	if (ret)
	94	return ret;
	95	code++;
e65f7ae7	96	} while (1);
9b960a38	97
40b53b77 MH	98	s3 = code;
40b53b77 MH	99	stage3:
9b960a38 MH	100	/* 3rd stage: store value to buffer */
9b960a38 MH	101	if (unlikely(!dest)) {
40b53b77	102	if (code->op == FETCH_OP_ST_STRING) {
489fe009	103	ret = fetch_store_strlen(val + code->offset);
40b53b77 MH	104	code++;
40b53b77 MH	105	goto array;
88903c46 MH	106	} else if (code->op == FETCH_OP_ST_USTRING) {
	107	ret += fetch_store_strlen_user(val + code->offset);
	108	code++;
	109	goto array;
40b53b77	110	} else
9b960a38 MH	111	return -EILSEQ;
	112	}
	113
	114	switch (code->op) {
	115	case FETCH_OP_ST_RAW:
	116	fetch_store_raw(val, code, dest);
	117	break;
	118	case FETCH_OP_ST_MEM:
	119	probe_mem_read(dest, (void *)val + code->offset, code->size);
	120	break;
e65f7ae7 MH	121	case FETCH_OP_ST_UMEM:
	122	probe_mem_read_user(dest, (void *)val + code->offset, code->size);
	123	break;
9b960a38	124	case FETCH_OP_ST_STRING:
40b53b77	125	loc = (u32 )dest;
9b960a38 MH	126	ret = fetch_store_string(val + code->offset, dest, base);
9b960a38 MH	127	break;
88903c46 MH	128	case FETCH_OP_ST_USTRING:
	129	loc = (u32 )dest;
	130	ret = fetch_store_string_user(val + code->offset, dest, base);
	131	break;
9b960a38 MH	132	default:
	133	return -EILSEQ;
	134	}
	135	code++;
	136
	137	/* 4th stage: modify stored value if needed */
	138	if (code->op == FETCH_OP_MOD_BF) {
	139	fetch_apply_bitfield(code, dest);
	140	code++;
	141	}
	142
40b53b77 MH	143	array:
	144	/* the last stage: Loop on array */
	145	if (code->op == FETCH_OP_LP_ARRAY) {
	146	total += ret;
	147	if (++i < code->param) {
	148	code = s3;
88903c46 MH	149	if (s3->op != FETCH_OP_ST_STRING &&
88903c46 MH	150	s3->op != FETCH_OP_ST_USTRING) {
40b53b77 MH	151	dest += s3->size;
	152	val += s3->size;
	153	goto stage3;
	154	}
	155	code--;
	156	val = lval + sizeof(char *);
	157	if (dest) {
	158	dest += sizeof(u32);
	159	(u32 )dest = update_data_loc(loc, ret);
	160	}
	161	goto stage2;
	162	}
	163	code++;
	164	ret = total;
	165	}
	166
9b960a38 MH	167	return code->op == FETCH_OP_END ? ret : -EILSEQ;
9b960a38 MH	168	}
53305928 MH	169
	170	/* Sum up total data length for dynamic arraies (strings) */
	171	static nokprobe_inline int
	172	__get_data_size(struct trace_probe tp, struct pt_regs regs)
	173	{
	174	struct probe_arg *arg;
9178412d	175	int i, len, ret = 0;
53305928 MH	176
	177	for (i = 0; i < tp->nr_args; i++) {
	178	arg = tp->args + i;
	179	if (unlikely(arg->dynamic)) {
9178412d MH	180	len = process_fetch_insn(arg->code, regs, NULL, NULL);
	181	if (len > 0)
	182	ret += len;
53305928 MH	183	}
	184	}
	185
	186	return ret;
	187	}
	188
	189	/* Store the value of each argument */
	190	static nokprobe_inline void
9178412d MH	191	store_trace_args(void data, struct trace_probe tp, struct pt_regs *regs,
9178412d MH	192	int header_size, int maxlen)
53305928 MH	193	{
53305928 MH	194	struct probe_arg *arg;
9178412d MH	195	void *base = data - header_size;
	196	void *dyndata = data + tp->size;
	197	u32 dl; / Data location */
	198	int ret, i;
53305928 MH	199
	200	for (i = 0; i < tp->nr_args; i++) {
	201	arg = tp->args + i;
9178412d MH	202	dl = data + arg->offset;
	203	/* Point the dynamic data area if needed */
	204	if (unlikely(arg->dynamic))
	205	*dl = make_data_loc(maxlen, dyndata - base);
	206	ret = process_fetch_insn(arg->code, regs, dl, base);
f6675872	207	if (unlikely(ret < 0 && arg->dynamic)) {
9178412d	208	*dl = make_data_loc(0, dyndata - base);
f6675872	209	} else {
9178412d	210	dyndata += ret;
f6675872 AZ	211	maxlen -= ret;
f6675872 AZ	212	}
53305928 MH	213	}
	214	}
	215
	216	static inline int
	217	print_probe_args(struct trace_seq s, struct probe_arg args, int nr_args,
	218	u8 data, void field)
	219	{
40b53b77 MH	220	void *p;
40b53b77 MH	221	int i, j;
53305928 MH	222
53305928 MH	223	for (i = 0; i < nr_args; i++) {
40b53b77 MH	224	struct probe_arg *a = args + i;
	225
	226	trace_seq_printf(s, " %s=", a->name);
	227	if (likely(!a->count)) {
	228	if (!a->type->print(s, data + a->offset, field))
	229	return -ENOMEM;
	230	continue;
	231	}
	232	trace_seq_putc(s, '{');
	233	p = data + a->offset;
	234	for (j = 0; j < a->count; j++) {
	235	if (!a->type->print(s, p, field))
	236	return -ENOMEM;
	237	trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
	238	p += a->type->size;
	239	}
53305928 MH	240	}
	241	return 0;
	242	}