[linux.git] / mm / maccess.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Access kernel or user memory without faulting.
 */
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/uaccess.h>

bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
		size_t size)
{
	return true;
}

#ifdef HAVE_GET_KERNEL_NOFAULT

#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)	\
	while (len >= sizeof(type)) {					\
		__get_kernel_nofault(dst, src, type, err_label);		\
		dst += sizeof(type);					\
		src += sizeof(type);					\
		len -= sizeof(type);					\
	}

long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
{
	if (!copy_from_kernel_nofault_allowed(src, size))
		return -ERANGE;

	pagefault_disable();
	copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
	copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
	copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
	copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
	pagefault_enable();
	return 0;
Efault:
	pagefault_enable();
	return -EFAULT;
}
EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);

#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)	\
	while (len >= sizeof(type)) {					\
		__put_kernel_nofault(dst, src, type, err_label);		\
		dst += sizeof(type);					\
		src += sizeof(type);					\
		len -= sizeof(type);					\
	}

long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
{
	pagefault_disable();
	copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
	copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
	copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
	copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
	pagefault_enable();
	return 0;
Efault:
	pagefault_enable();
	return -EFAULT;
}

long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
{
	const void *src = unsafe_addr;

	if (unlikely(count <= 0))
		return 0;
	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
		return -ERANGE;

	pagefault_disable();
	do {
		__get_kernel_nofault(dst, src, u8, Efault);
		dst++;
		src++;
	} while (dst[-1] && src - unsafe_addr < count);
	pagefault_enable();

	dst[-1] = '\0';
	return src - unsafe_addr;
Efault:
	pagefault_enable();
	dst[-1] = '\0';
	return -EFAULT;
}
#else /* HAVE_GET_KERNEL_NOFAULT */
/**
 * copy_from_kernel_nofault(): safely attempt to read from kernel-space
 * @dst: pointer to the buffer that shall take the data
 * @src: address to read from
 * @size: size of the data chunk
 *
 * Safely read from kernel address @src to the buffer at @dst.  If a kernel
 * fault happens, handle that and return -EFAULT.  If @src is not a valid kernel
 * address, return -ERANGE.
 *
 * We ensure that the copy_from_user is executed in atomic context so that
 * do_page_fault() doesn't attempt to take mmap_lock.  This makes
 * copy_from_kernel_nofault() suitable for use within regions where the caller
 * already holds mmap_lock, or other locks which nest inside mmap_lock.
 */
long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
{
	long ret;
	mm_segment_t old_fs = get_fs();

	if (!copy_from_kernel_nofault_allowed(src, size))
		return -ERANGE;

	set_fs(KERNEL_DS);
	pagefault_disable();
	ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
			size);
	pagefault_enable();
	set_fs(old_fs);

	if (ret)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);

/**
 * copy_to_kernel_nofault(): safely attempt to write to a location
 * @dst: address to write to
 * @src: pointer to the data that shall be written
 * @size: size of the data chunk
 *
 * Safely write to address @dst from the buffer at @src.  If a kernel fault
 * happens, handle that and return -EFAULT.
 */
long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
{
	long ret;
	mm_segment_t old_fs = get_fs();

	set_fs(KERNEL_DS);
	pagefault_disable();
	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
	pagefault_enable();
	set_fs(old_fs);

	if (ret)
		return -EFAULT;
	return 0;
}

/**
 * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
 *				 address.
 * @dst:   Destination address, in kernel space.  This buffer must be at
 *         least @count bytes long.
 * @unsafe_addr: Unsafe address.
 * @count: Maximum number of bytes to copy, including the trailing NUL.
 *
 * Copies a NUL-terminated string from unsafe address to kernel buffer.
 *
 * On success, returns the length of the string INCLUDING the trailing NUL.
 *
 * If access fails, returns -EFAULT (some data may have been copied and the
 * trailing NUL added).  If @unsafe_addr is not a valid kernel address, return
 * -ERANGE.
 *
 * If @count is smaller than the length of the string, copies @count-1 bytes,
 * sets the last byte of @dst buffer to NUL and returns @count.
 */
long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
{
	mm_segment_t old_fs = get_fs();
	const void *src = unsafe_addr;
	long ret;

	if (unlikely(count <= 0))
		return 0;
	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
		return -ERANGE;

	set_fs(KERNEL_DS);
	pagefault_disable();

	do {
		ret = __get_user(*dst++, (const char __user __force *)src++);
	} while (dst[-1] && ret == 0 && src - unsafe_addr < count);

	dst[-1] = '\0';
	pagefault_enable();
	set_fs(old_fs);

	return ret ? -EFAULT : src - unsafe_addr;
}
#endif /* HAVE_GET_KERNEL_NOFAULT */

/**
 * copy_from_user_nofault(): safely attempt to read from a user-space location
 * @dst: pointer to the buffer that shall take the data
 * @src: address to read from. This must be a user address.
 * @size: size of the data chunk
 *
 * Safely read from user address @src to the buffer at @dst. If a kernel fault
 * happens, handle that and return -EFAULT.
 */
long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
{
	long ret = -EFAULT;
	mm_segment_t old_fs = force_uaccess_begin();

	if (access_ok(src, size)) {
		pagefault_disable();
		ret = __copy_from_user_inatomic(dst, src, size);
		pagefault_enable();
	}
	force_uaccess_end(old_fs);

	if (ret)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(copy_from_user_nofault);

/**
 * copy_to_user_nofault(): safely attempt to write to a user-space location
 * @dst: address to write to
 * @src: pointer to the data that shall be written
 * @size: size of the data chunk
 *
 * Safely write to address @dst from the buffer at @src.  If a kernel fault
 * happens, handle that and return -EFAULT.
 */
long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
{
	long ret = -EFAULT;
	mm_segment_t old_fs = force_uaccess_begin();

	if (access_ok(dst, size)) {
		pagefault_disable();
		ret = __copy_to_user_inatomic(dst, src, size);
		pagefault_enable();
	}
	force_uaccess_end(old_fs);

	if (ret)
		return -EFAULT;
	return 0;
}
EXPORT_SYMBOL_GPL(copy_to_user_nofault);

/**
 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
 *				address.
 * @dst:   Destination address, in kernel space.  This buffer must be at
 *         least @count bytes long.
 * @unsafe_addr: Unsafe user address.
 * @count: Maximum number of bytes to copy, including the trailing NUL.
 *
 * Copies a NUL-terminated string from unsafe user address to kernel buffer.
 *
 * On success, returns the length of the string INCLUDING the trailing NUL.
 *
 * If access fails, returns -EFAULT (some data may have been copied
 * and the trailing NUL added).
 *
 * If @count is smaller than the length of the string, copies @count-1 bytes,
 * sets the last byte of @dst buffer to NUL and returns @count.
 */
long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
			      long count)
{
	mm_segment_t old_fs;
	long ret;

	if (unlikely(count <= 0))
		return 0;

	old_fs = force_uaccess_begin();
	pagefault_disable();
	ret = strncpy_from_user(dst, unsafe_addr, count);
	pagefault_enable();
	force_uaccess_end(old_fs);

	if (ret >= count) {
		ret = count;
		dst[ret - 1] = '\0';
	} else if (ret > 0) {
		ret++;
	}

	return ret;
}

/**
 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
 * @unsafe_addr: The string to measure.
 * @count: Maximum count (including NUL)
 *
 * Get the size of a NUL-terminated string in user space without pagefault.
 *
 * Returns the size of the string INCLUDING the terminating NUL.
 *
 * If the string is too long, returns a number larger than @count. User
 * has to check the return value against "> count".
 * On exception (or invalid count), returns 0.
 *
 * Unlike strnlen_user, this can be used from IRQ handler etc. because
 * it disables pagefaults.
 */
long strnlen_user_nofault(const void __user *unsafe_addr, long count)
{
	mm_segment_t old_fs;
	int ret;

	old_fs = force_uaccess_begin();
	pagefault_disable();
	ret = strnlen_user(unsafe_addr, count);
	pagefault_enable();
	force_uaccess_end(old_fs);

	return ret;
}
Commit	Line	Data
457c8996	1	// SPDX-License-Identifier: GPL-2.0-only
c33fa9f5	2	/*
3f0acb1e	3	* Access kernel or user memory without faulting.
c33fa9f5	4	*/
b95f1b31	5	#include <linux/export.h>
c33fa9f5	6	#include <linux/mm.h>
7c7fcf76	7	#include <linux/uaccess.h>
c33fa9f5	8
fe557319 CH	9	bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
fe557319 CH	10	size_t size)
eab0c608 CH	11	{
	12	return true;
	13	}
	14
b58294ea CH	15	#ifdef HAVE_GET_KERNEL_NOFAULT
b58294ea CH	16
fe557319	17	#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
b58294ea CH	18	while (len >= sizeof(type)) { \
	19	__get_kernel_nofault(dst, src, type, err_label); \
	20	dst += sizeof(type); \
	21	src += sizeof(type); \
	22	len -= sizeof(type); \
	23	}
	24
fe557319	25	long copy_from_kernel_nofault(void dst, const void src, size_t size)
b58294ea	26	{
fe557319	27	if (!copy_from_kernel_nofault_allowed(src, size))
2a71e81d	28	return -ERANGE;
b58294ea CH	29
b58294ea CH	30	pagefault_disable();
fe557319 CH	31	copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
	32	copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
	33	copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
	34	copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
b58294ea CH	35	pagefault_enable();
	36	return 0;
	37	Efault:
	38	pagefault_enable();
	39	return -EFAULT;
	40	}
fe557319	41	EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
b58294ea	42
fe557319	43	#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
b58294ea CH	44	while (len >= sizeof(type)) { \
	45	__put_kernel_nofault(dst, src, type, err_label); \
	46	dst += sizeof(type); \
	47	src += sizeof(type); \
	48	len -= sizeof(type); \
	49	}
	50
fe557319	51	long copy_to_kernel_nofault(void dst, const void src, size_t size)
b58294ea CH	52	{
b58294ea CH	53	pagefault_disable();
fe557319 CH	54	copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
	55	copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
	56	copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
	57	copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
b58294ea CH	58	pagefault_enable();
	59	return 0;
	60	Efault:
	61	pagefault_enable();
	62	return -EFAULT;
	63	}
	64
	65	long strncpy_from_kernel_nofault(char dst, const void unsafe_addr, long count)
	66	{
	67	const void *src = unsafe_addr;
	68
	69	if (unlikely(count <= 0))
	70	return 0;
fe557319	71	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
2a71e81d	72	return -ERANGE;
b58294ea CH	73
	74	pagefault_disable();
	75	do {
	76	__get_kernel_nofault(dst, src, u8, Efault);
	77	dst++;
	78	src++;
	79	} while (dst[-1] && src - unsafe_addr < count);
	80	pagefault_enable();
	81
	82	dst[-1] = '\0';
	83	return src - unsafe_addr;
	84	Efault:
	85	pagefault_enable();
	86	dst[-1] = '\0';
	87	return -EFAULT;
	88	}
	89	#else /* HAVE_GET_KERNEL_NOFAULT */
c33fa9f5	90	/**
fe557319	91	* copy_from_kernel_nofault(): safely attempt to read from kernel-space
4f6de12b CH	92	* @dst: pointer to the buffer that shall take the data
	93	* @src: address to read from
	94	* @size: size of the data chunk
	95	*
	96	* Safely read from kernel address @src to the buffer at @dst. If a kernel
2a71e81d CH	97	* fault happens, handle that and return -EFAULT. If @src is not a valid kernel
2a71e81d CH	98	* address, return -ERANGE.
0ab32b6f AM	99	*
0ab32b6f AM	100	* We ensure that the copy_from_user is executed in atomic context so that
c1e8d7c6	101	* do_page_fault() doesn't attempt to take mmap_lock. This makes
fe557319	102	* copy_from_kernel_nofault() suitable for use within regions where the caller
c1e8d7c6	103	* already holds mmap_lock, or other locks which nest inside mmap_lock.
c33fa9f5	104	*/
fe557319	105	long copy_from_kernel_nofault(void dst, const void src, size_t size)
c33fa9f5 IM	106	{
c33fa9f5 IM	107	long ret;
b4b8ac52	108	mm_segment_t old_fs = get_fs();
c33fa9f5	109
fe557319	110	if (!copy_from_kernel_nofault_allowed(src, size))
2a71e81d	111	return -ERANGE;
eab0c608	112
b4b8ac52	113	set_fs(KERNEL_DS);
cd030905 CH	114	pagefault_disable();
	115	ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
	116	size);
	117	pagefault_enable();
b4b8ac52	118	set_fs(old_fs);
c33fa9f5	119
cd030905 CH	120	if (ret)
	121	return -EFAULT;
	122	return 0;
c33fa9f5	123	}
fe557319	124	EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
c33fa9f5 IM	125
c33fa9f5 IM	126	/**
fe557319	127	* copy_to_kernel_nofault(): safely attempt to write to a location
c33fa9f5 IM	128	* @dst: address to write to
	129	* @src: pointer to the data that shall be written
	130	* @size: size of the data chunk
	131	*
	132	* Safely write to address @dst from the buffer at @src. If a kernel fault
	133	* happens, handle that and return -EFAULT.
	134	*/
fe557319	135	long copy_to_kernel_nofault(void dst, const void src, size_t size)
c33fa9f5 IM	136	{
c33fa9f5 IM	137	long ret;
b4b8ac52	138	mm_segment_t old_fs = get_fs();
c33fa9f5	139
b4b8ac52	140	set_fs(KERNEL_DS);
cd030905 CH	141	pagefault_disable();
	142	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
	143	pagefault_enable();
b4b8ac52	144	set_fs(old_fs);
c33fa9f5	145
cd030905 CH	146	if (ret)
	147	return -EFAULT;
	148	return 0;
c33fa9f5	149	}
dbb7ee0e	150
4f6de12b	151	/**
c4cb1644	152	* strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
4f6de12b CH	153	* address.
	154	* @dst: Destination address, in kernel space. This buffer must be at
	155	* least @count bytes long.
	156	* @unsafe_addr: Unsafe address.
	157	* @count: Maximum number of bytes to copy, including the trailing NUL.
	158	*
	159	* Copies a NUL-terminated string from unsafe address to kernel buffer.
	160	*
	161	* On success, returns the length of the string INCLUDING the trailing NUL.
	162	*
2a71e81d CH	163	* If access fails, returns -EFAULT (some data may have been copied and the
	164	* trailing NUL added). If @unsafe_addr is not a valid kernel address, return
	165	* -ERANGE.
4f6de12b CH	166	*
	167	* If @count is smaller than the length of the string, copies @count-1 bytes,
	168	* sets the last byte of @dst buffer to NUL and returns @count.
	169	*/
eab0c608	170	long strncpy_from_kernel_nofault(char dst, const void unsafe_addr, long count)
dbb7ee0e AS	171	{
	172	mm_segment_t old_fs = get_fs();
	173	const void *src = unsafe_addr;
	174	long ret;
	175
	176	if (unlikely(count <= 0))
	177	return 0;
fe557319	178	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
2a71e81d	179	return -ERANGE;
dbb7ee0e AS	180
	181	set_fs(KERNEL_DS);
	182	pagefault_disable();
	183
	184	do {
bd28b145	185	ret = __get_user(dst++, (const char __user __force )src++);
dbb7ee0e AS	186	} while (dst[-1] && ret == 0 && src - unsafe_addr < count);
	187
	188	dst[-1] = '\0';
	189	pagefault_enable();
	190	set_fs(old_fs);
	191
9dd861d5	192	return ret ? -EFAULT : src - unsafe_addr;
dbb7ee0e	193	}
b58294ea	194	#endif /* HAVE_GET_KERNEL_NOFAULT */
3d708182	195
fc3562d7	196	/**
c0ee37e8	197	* copy_from_user_nofault(): safely attempt to read from a user-space location
fc3562d7 CH	198	* @dst: pointer to the buffer that shall take the data
	199	* @src: address to read from. This must be a user address.
	200	* @size: size of the data chunk
	201	*
	202	* Safely read from user address @src to the buffer at @dst. If a kernel fault
	203	* happens, handle that and return -EFAULT.
	204	*/
c0ee37e8	205	long copy_from_user_nofault(void dst, const void __user src, size_t size)
fc3562d7 CH	206	{
fc3562d7 CH	207	long ret = -EFAULT;
3d13f313	208	mm_segment_t old_fs = force_uaccess_begin();
fc3562d7	209
fc3562d7 CH	210	if (access_ok(src, size)) {
	211	pagefault_disable();
	212	ret = __copy_from_user_inatomic(dst, src, size);
	213	pagefault_enable();
	214	}
3d13f313	215	force_uaccess_end(old_fs);
fc3562d7 CH	216
	217	if (ret)
	218	return -EFAULT;
	219	return 0;
	220	}
c0ee37e8	221	EXPORT_SYMBOL_GPL(copy_from_user_nofault);
fc3562d7 CH	222
fc3562d7 CH	223	/**
c0ee37e8	224	* copy_to_user_nofault(): safely attempt to write to a user-space location
fc3562d7 CH	225	* @dst: address to write to
	226	* @src: pointer to the data that shall be written
	227	* @size: size of the data chunk
	228	*
	229	* Safely write to address @dst from the buffer at @src. If a kernel fault
	230	* happens, handle that and return -EFAULT.
	231	*/
c0ee37e8	232	long copy_to_user_nofault(void __user dst, const void src, size_t size)
fc3562d7 CH	233	{
fc3562d7 CH	234	long ret = -EFAULT;
3d13f313	235	mm_segment_t old_fs = force_uaccess_begin();
fc3562d7	236
fc3562d7 CH	237	if (access_ok(dst, size)) {
	238	pagefault_disable();
	239	ret = __copy_to_user_inatomic(dst, src, size);
	240	pagefault_enable();
	241	}
3d13f313	242	force_uaccess_end(old_fs);
fc3562d7 CH	243
	244	if (ret)
	245	return -EFAULT;
	246	return 0;
	247	}
c0ee37e8	248	EXPORT_SYMBOL_GPL(copy_to_user_nofault);
fc3562d7	249
3d708182	250	/**
bd88bb5d	251	* strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
3d708182 MH	252	* address.
	253	* @dst: Destination address, in kernel space. This buffer must be at
	254	* least @count bytes long.
	255	* @unsafe_addr: Unsafe user address.
	256	* @count: Maximum number of bytes to copy, including the trailing NUL.
	257	*
	258	* Copies a NUL-terminated string from unsafe user address to kernel buffer.
	259	*
	260	* On success, returns the length of the string INCLUDING the trailing NUL.
	261	*
	262	* If access fails, returns -EFAULT (some data may have been copied
	263	* and the trailing NUL added).
	264	*
	265	* If @count is smaller than the length of the string, copies @count-1 bytes,
	266	* sets the last byte of @dst buffer to NUL and returns @count.
	267	*/
bd88bb5d	268	long strncpy_from_user_nofault(char dst, const void __user unsafe_addr,
3d708182 MH	269	long count)
3d708182 MH	270	{
3d13f313	271	mm_segment_t old_fs;
3d708182 MH	272	long ret;
	273
	274	if (unlikely(count <= 0))
	275	return 0;
	276
3d13f313	277	old_fs = force_uaccess_begin();
3d708182 MH	278	pagefault_disable();
	279	ret = strncpy_from_user(dst, unsafe_addr, count);
	280	pagefault_enable();
3d13f313	281	force_uaccess_end(old_fs);
3d708182 MH	282
	283	if (ret >= count) {
	284	ret = count;
	285	dst[ret - 1] = '\0';
	286	} else if (ret > 0) {
	287	ret++;
	288	}
	289
	290	return ret;
	291	}
	292
	293	/**
02dddb16	294	* strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
3d708182 MH	295	* @unsafe_addr: The string to measure.
	296	* @count: Maximum count (including NUL)
	297	*
	298	* Get the size of a NUL-terminated string in user space without pagefault.
	299	*
	300	* Returns the size of the string INCLUDING the terminating NUL.
	301	*
	302	* If the string is too long, returns a number larger than @count. User
	303	* has to check the return value against "> count".
	304	* On exception (or invalid count), returns 0.
	305	*
	306	* Unlike strnlen_user, this can be used from IRQ handler etc. because
	307	* it disables pagefaults.
	308	*/
02dddb16	309	long strnlen_user_nofault(const void __user *unsafe_addr, long count)
3d708182	310	{
3d13f313	311	mm_segment_t old_fs;
3d708182 MH	312	int ret;
3d708182 MH	313
3d13f313	314	old_fs = force_uaccess_begin();
3d708182 MH	315	pagefault_disable();
	316	ret = strnlen_user(unsafe_addr, count);
	317	pagefault_enable();
3d13f313	318	force_uaccess_end(old_fs);
3d708182 MH	319
	320	return ret;
	321	}