[linux.git] / kernel / spinlock.c

/*
 * Copyright (2004) Linus Torvalds
 *
 * Author: Zwane Mwaikambo <[email protected]>
 *
 * Copyright (2004, 2005) Ingo Molnar
 *
 * This file contains the spinlock/rwlock implementations for the
 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
 *
 * Note that some architectures have special knowledge about the
 * stack frames of these functions in their profile_pc. If you
 * change anything significant here that could change the stack
 * frame contact the architecture maintainers.
 */

#include <linux/linkage.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/debug_locks.h>
#include <linux/module.h>

int __lockfunc _spin_trylock(spinlock_t *lock)
{
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}
	
	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_spin_trylock);

int __lockfunc _read_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_read_trylock(lock)) {
		rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_read_trylock);

int __lockfunc _write_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_write_trylock(lock)) {
		rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_write_trylock);

/*
 * If lockdep is enabled then we use the non-preemption spin-ops
 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
 */
#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)

void __lockfunc _read_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
}
EXPORT_SYMBOL(_read_lock);

unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	/*
	 * On lockdep we dont want the hand-coded irq-enable of
	 * _raw_spin_lock_flags() code, because lockdep assumes
	 * that interrupts are not re-enabled during lock-acquire:
	 */
#ifdef CONFIG_LOCKDEP
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
#else
	_raw_spin_lock_flags(lock, &flags);
#endif
	return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave);

void __lockfunc _spin_lock_irq(spinlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_irq);

void __lockfunc _spin_lock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_bh);

unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
			     _raw_read_lock_flags, &flags);
	return flags;
}
EXPORT_SYMBOL(_read_lock_irqsave);

void __lockfunc _read_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
}
EXPORT_SYMBOL(_read_lock_irq);

void __lockfunc _read_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
}
EXPORT_SYMBOL(_read_lock_bh);

unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
			     _raw_write_lock_flags, &flags);
	return flags;
}
EXPORT_SYMBOL(_write_lock_irqsave);

void __lockfunc _write_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
}
EXPORT_SYMBOL(_write_lock_irq);

void __lockfunc _write_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
}
EXPORT_SYMBOL(_write_lock_bh);

void __lockfunc _spin_lock(spinlock_t *lock)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}

EXPORT_SYMBOL(_spin_lock);

void __lockfunc _write_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
}

EXPORT_SYMBOL(_write_lock);

#else /* CONFIG_PREEMPT: */

/*
 * This could be a long-held lock. We both prepare to spin for a long
 * time (making _this_ CPU preemptable if possible), and we also signal
 * towards that other CPU that it should break the lock ASAP.
 *
 * (We do this in a function because inlining it would be excessive.)
 */

#define BUILD_LOCK_OPS(op, locktype)					\
void __lockfunc _##op##_lock(locktype##_t *lock)			\
{									\
	for (;;) {							\
		preempt_disable();					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			_raw_##op##_relax(&lock->raw_lock);		\
	}								\
	(lock)->break_lock = 0;						\
}									\
									\
EXPORT_SYMBOL(_##op##_lock);						\
									\
unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)	\
{									\
	unsigned long flags;						\
									\
	for (;;) {							\
		preempt_disable();					\
		local_irq_save(flags);					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		local_irq_restore(flags);				\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			_raw_##op##_relax(&lock->raw_lock);		\
	}								\
	(lock)->break_lock = 0;						\
	return flags;							\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irqsave);					\
									\
void __lockfunc _##op##_lock_irq(locktype##_t *lock)			\
{									\
	_##op##_lock_irqsave(lock);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irq);					\
									\
void __lockfunc _##op##_lock_bh(locktype##_t *lock)			\
{									\
	unsigned long flags;						\
									\
	/*							*/	\
	/* Careful: we must exclude softirqs too, hence the	*/	\
	/* irq-disabling. We use the generic preemption-aware	*/	\
	/* function:						*/	\
	/**/								\
	flags = _##op##_lock_irqsave(lock);				\
	local_bh_disable();						\
	local_irq_restore(flags);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_bh)

/*
 * Build preemption-friendly versions of the following
 * lock-spinning functions:
 *
 *         _[spin|read|write]_lock()
 *         _[spin|read|write]_lock_irq()
 *         _[spin|read|write]_lock_irqsave()
 *         _[spin|read|write]_lock_bh()
 */
BUILD_LOCK_OPS(spin, spinlock);
BUILD_LOCK_OPS(read, rwlock);
BUILD_LOCK_OPS(write, rwlock);

#endif /* CONFIG_PREEMPT */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_nested);

unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
				_raw_spin_lock_flags, &flags);
	return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave_nested);

void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
				     struct lockdep_map *nest_lock)
{
	preempt_disable();
	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
}
EXPORT_SYMBOL(_spin_lock_nest_lock);

#endif

void __lockfunc _spin_unlock(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock);

void __lockfunc _write_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock);

void __lockfunc _read_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock);

void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irqrestore);

void __lockfunc _spin_unlock_irq(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irq);

void __lockfunc _spin_unlock_bh(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_spin_unlock_bh);

void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irqrestore);

void __lockfunc _read_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irq);

void __lockfunc _read_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_read_unlock_bh);

void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irqrestore);

void __lockfunc _write_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irq);

void __lockfunc _write_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_write_unlock_bh);

int __lockfunc _spin_trylock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	return 0;
}
EXPORT_SYMBOL(_spin_trylock_bh);

notrace int in_lock_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __lockfunc functions */
	extern char __lock_text_start[], __lock_text_end[];

	return addr >= (unsigned long)__lock_text_start
	&& addr < (unsigned long)__lock_text_end;
}
EXPORT_SYMBOL(in_lock_functions);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Copyright (2004) Linus Torvalds
	3	*
	4	* Author: Zwane Mwaikambo <[email protected]>
	5	*
fb1c8f93 IM	6	* Copyright (2004, 2005) Ingo Molnar
	7	*
	8	* This file contains the spinlock/rwlock implementations for the
	9	* SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
0cb91a22 AK	10	*
	11	* Note that some architectures have special knowledge about the
	12	* stack frames of these functions in their profile_pc. If you
	13	* change anything significant here that could change the stack
	14	* frame contact the architecture maintainers.
1da177e4 LT	15	*/
1da177e4 LT	16
1da177e4 LT	17	#include <linux/linkage.h>
	18	#include <linux/preempt.h>
	19	#include <linux/spinlock.h>
	20	#include <linux/interrupt.h>
8a25d5de	21	#include <linux/debug_locks.h>
1da177e4 LT	22	#include <linux/module.h>
1da177e4 LT	23
1da177e4 LT	24	int __lockfunc _spin_trylock(spinlock_t *lock)
	25	{
	26	preempt_disable();
8a25d5de IM	27	if (_raw_spin_trylock(lock)) {
8a25d5de IM	28	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	29	return 1;
8a25d5de	30	}
1da177e4 LT	31
	32	preempt_enable();
	33	return 0;
	34	}
	35	EXPORT_SYMBOL(_spin_trylock);
	36
	37	int __lockfunc _read_trylock(rwlock_t *lock)
	38	{
	39	preempt_disable();
8a25d5de IM	40	if (_raw_read_trylock(lock)) {
8a25d5de IM	41	rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	42	return 1;
8a25d5de	43	}
1da177e4 LT	44
	45	preempt_enable();
	46	return 0;
	47	}
	48	EXPORT_SYMBOL(_read_trylock);
	49
	50	int __lockfunc _write_trylock(rwlock_t *lock)
	51	{
	52	preempt_disable();
8a25d5de IM	53	if (_raw_write_trylock(lock)) {
8a25d5de IM	54	rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	55	return 1;
8a25d5de	56	}
1da177e4 LT	57
	58	preempt_enable();
	59	return 0;
	60	}
	61	EXPORT_SYMBOL(_write_trylock);
	62
8a25d5de IM	63	/*
	64	* If lockdep is enabled then we use the non-preemption spin-ops
	65	* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
	66	* not re-enabled during lock-acquire (which the preempt-spin-ops do):
	67	*/
95c354fe	68	#if !defined(CONFIG_GENERIC_LOCKBREAK) \|\| defined(CONFIG_DEBUG_LOCK_ALLOC)
1da177e4 LT	69
	70	void __lockfunc _read_lock(rwlock_t *lock)
	71	{
	72	preempt_disable();
8a25d5de	73	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	74	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
1da177e4 LT	75	}
	76	EXPORT_SYMBOL(_read_lock);
	77
	78	unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
	79	{
	80	unsigned long flags;
	81
	82	local_irq_save(flags);
	83	preempt_disable();
8a25d5de IM	84	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	85	/*
	86	* On lockdep we dont want the hand-coded irq-enable of
	87	* _raw_spin_lock_flags() code, because lockdep assumes
	88	* that interrupts are not re-enabled during lock-acquire:
	89	*/
ca58abcb	90	#ifdef CONFIG_LOCKDEP
4fe87745	91	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
8a25d5de	92	#else
fb1c8f93	93	_raw_spin_lock_flags(lock, &flags);
8a25d5de	94	#endif
1da177e4 LT	95	return flags;
	96	}
	97	EXPORT_SYMBOL(_spin_lock_irqsave);
	98
	99	void __lockfunc _spin_lock_irq(spinlock_t *lock)
	100	{
	101	local_irq_disable();
	102	preempt_disable();
8a25d5de	103	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	104	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
1da177e4 LT	105	}
	106	EXPORT_SYMBOL(_spin_lock_irq);
	107
	108	void __lockfunc _spin_lock_bh(spinlock_t *lock)
	109	{
	110	local_bh_disable();
	111	preempt_disable();
8a25d5de	112	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	113	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
1da177e4 LT	114	}
	115	EXPORT_SYMBOL(_spin_lock_bh);
	116
	117	unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
	118	{
	119	unsigned long flags;
	120
	121	local_irq_save(flags);
	122	preempt_disable();
8a25d5de	123	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
f5f7eac4 RH	124	LOCK_CONTENDED_FLAGS(lock, _raw_read_trylock, _raw_read_lock,
f5f7eac4 RH	125	_raw_read_lock_flags, &flags);
1da177e4 LT	126	return flags;
	127	}
	128	EXPORT_SYMBOL(_read_lock_irqsave);
	129
	130	void __lockfunc _read_lock_irq(rwlock_t *lock)
	131	{
	132	local_irq_disable();
	133	preempt_disable();
8a25d5de	134	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	135	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
1da177e4 LT	136	}
	137	EXPORT_SYMBOL(_read_lock_irq);
	138
	139	void __lockfunc _read_lock_bh(rwlock_t *lock)
	140	{
	141	local_bh_disable();
	142	preempt_disable();
8a25d5de	143	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	144	LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
1da177e4 LT	145	}
	146	EXPORT_SYMBOL(_read_lock_bh);
	147
	148	unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
	149	{
	150	unsigned long flags;
	151
	152	local_irq_save(flags);
	153	preempt_disable();
8a25d5de	154	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
f5f7eac4 RH	155	LOCK_CONTENDED_FLAGS(lock, _raw_write_trylock, _raw_write_lock,
f5f7eac4 RH	156	_raw_write_lock_flags, &flags);
1da177e4 LT	157	return flags;
	158	}
	159	EXPORT_SYMBOL(_write_lock_irqsave);
	160
	161	void __lockfunc _write_lock_irq(rwlock_t *lock)
	162	{
	163	local_irq_disable();
	164	preempt_disable();
8a25d5de	165	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	166	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
1da177e4 LT	167	}
	168	EXPORT_SYMBOL(_write_lock_irq);
	169
	170	void __lockfunc _write_lock_bh(rwlock_t *lock)
	171	{
	172	local_bh_disable();
	173	preempt_disable();
8a25d5de	174	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	175	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
1da177e4 LT	176	}
	177	EXPORT_SYMBOL(_write_lock_bh);
	178
	179	void __lockfunc _spin_lock(spinlock_t *lock)
	180	{
	181	preempt_disable();
8a25d5de	182	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	183	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
1da177e4 LT	184	}
	185
	186	EXPORT_SYMBOL(_spin_lock);
	187
	188	void __lockfunc _write_lock(rwlock_t *lock)
	189	{
	190	preempt_disable();
8a25d5de	191	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
4fe87745	192	LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
1da177e4 LT	193	}
	194
	195	EXPORT_SYMBOL(_write_lock);
	196
	197	#else /* CONFIG_PREEMPT: */
	198
	199	/*
	200	* This could be a long-held lock. We both prepare to spin for a long
	201	* time (making _this_ CPU preemptable if possible), and we also signal
	202	* towards that other CPU that it should break the lock ASAP.
	203	*
	204	* (We do this in a function because inlining it would be excessive.)
	205	*/
	206
	207	#define BUILD_LOCK_OPS(op, locktype) \
	208	void __lockfunc _##op##_lock(locktype##_t *lock) \
	209	{ \
1da177e4	210	for (;;) { \
ee25e96f	211	preempt_disable(); \
1da177e4 LT	212	if (likely(_raw_##op##_trylock(lock))) \
	213	break; \
	214	preempt_enable(); \
ee25e96f	215	\
1da177e4 LT	216	if (!(lock)->break_lock) \
	217	(lock)->break_lock = 1; \
	218	while (!op##_can_lock(lock) && (lock)->break_lock) \
ef6edc97	219	_raw_##op##_relax(&lock->raw_lock); \
1da177e4 LT	220	} \
	221	(lock)->break_lock = 0; \
	222	} \
	223	\
	224	EXPORT_SYMBOL(_##op##_lock); \
	225	\
	226	unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
	227	{ \
	228	unsigned long flags; \
	229	\
1da177e4	230	for (;;) { \
ee25e96f	231	preempt_disable(); \
1da177e4 LT	232	local_irq_save(flags); \
	233	if (likely(_raw_##op##_trylock(lock))) \
	234	break; \
	235	local_irq_restore(flags); \
1da177e4	236	preempt_enable(); \
ee25e96f	237	\
1da177e4 LT	238	if (!(lock)->break_lock) \
	239	(lock)->break_lock = 1; \
	240	while (!op##_can_lock(lock) && (lock)->break_lock) \
ef6edc97	241	_raw_##op##_relax(&lock->raw_lock); \
1da177e4 LT	242	} \
	243	(lock)->break_lock = 0; \
	244	return flags; \
	245	} \
	246	\
	247	EXPORT_SYMBOL(_##op##_lock_irqsave); \
	248	\
	249	void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
	250	{ \
	251	_##op##_lock_irqsave(lock); \
	252	} \
	253	\
	254	EXPORT_SYMBOL(_##op##_lock_irq); \
	255	\
	256	void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
	257	{ \
	258	unsigned long flags; \
	259	\
	260	/* */ \
	261	/* Careful: we must exclude softirqs too, hence the */ \
	262	/* irq-disabling. We use the generic preemption-aware */ \
	263	/* function: */ \
	264	/**/ \
	265	flags = _##op##_lock_irqsave(lock); \
	266	local_bh_disable(); \
	267	local_irq_restore(flags); \
	268	} \
	269	\
	270	EXPORT_SYMBOL(_##op##_lock_bh)
	271
	272	/*
	273	* Build preemption-friendly versions of the following
	274	* lock-spinning functions:
	275	*
	276	* _[spin\|read\|write]_lock()
	277	* _[spin\|read\|write]_lock_irq()
	278	* _[spin\|read\|write]_lock_irqsave()
	279	* _[spin\|read\|write]_lock_bh()
	280	*/
	281	BUILD_LOCK_OPS(spin, spinlock);
	282	BUILD_LOCK_OPS(read, rwlock);
	283	BUILD_LOCK_OPS(write, rwlock);
	284
	285	#endif /* CONFIG_PREEMPT */
	286
8a25d5de IM	287	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	288
	289	void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
	290	{
	291	preempt_disable();
	292	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
4fe87745	293	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
8a25d5de	294	}
8a25d5de	295	EXPORT_SYMBOL(_spin_lock_nested);
b7d39aff	296
cfd3ef23 AV	297	unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
	298	{
	299	unsigned long flags;
	300
	301	local_irq_save(flags);
	302	preempt_disable();
	303	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
e8c158bb RH	304	LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
e8c158bb RH	305	_raw_spin_lock_flags, &flags);
cfd3ef23 AV	306	return flags;
cfd3ef23 AV	307	}
cfd3ef23	308	EXPORT_SYMBOL(_spin_lock_irqsave_nested);
8a25d5de	309
b7d39aff PZ	310	void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
	311	struct lockdep_map *nest_lock)
	312	{
	313	preempt_disable();
	314	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
	315	LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
	316	}
b7d39aff PZ	317	EXPORT_SYMBOL(_spin_lock_nest_lock);
b7d39aff PZ	318
8a25d5de IM	319	#endif
8a25d5de IM	320
1da177e4 LT	321	void __lockfunc _spin_unlock(spinlock_t *lock)
1da177e4 LT	322	{
8a25d5de	323	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	324	_raw_spin_unlock(lock);
	325	preempt_enable();
	326	}
	327	EXPORT_SYMBOL(_spin_unlock);
	328
	329	void __lockfunc _write_unlock(rwlock_t *lock)
	330	{
8a25d5de	331	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	332	_raw_write_unlock(lock);
	333	preempt_enable();
	334	}
	335	EXPORT_SYMBOL(_write_unlock);
	336
	337	void __lockfunc _read_unlock(rwlock_t *lock)
	338	{
8a25d5de	339	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	340	_raw_read_unlock(lock);
	341	preempt_enable();
	342	}
	343	EXPORT_SYMBOL(_read_unlock);
	344
	345	void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
	346	{
8a25d5de	347	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	348	_raw_spin_unlock(lock);
	349	local_irq_restore(flags);
	350	preempt_enable();
	351	}
	352	EXPORT_SYMBOL(_spin_unlock_irqrestore);
	353
	354	void __lockfunc _spin_unlock_irq(spinlock_t *lock)
	355	{
8a25d5de	356	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	357	_raw_spin_unlock(lock);
	358	local_irq_enable();
	359	preempt_enable();
	360	}
	361	EXPORT_SYMBOL(_spin_unlock_irq);
	362
	363	void __lockfunc _spin_unlock_bh(spinlock_t *lock)
	364	{
8a25d5de	365	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	366	_raw_spin_unlock(lock);
10f02d1c	367	preempt_enable_no_resched();
8a25d5de	368	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	369	}
	370	EXPORT_SYMBOL(_spin_unlock_bh);
	371
	372	void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	373	{
8a25d5de	374	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	375	_raw_read_unlock(lock);
	376	local_irq_restore(flags);
	377	preempt_enable();
	378	}
	379	EXPORT_SYMBOL(_read_unlock_irqrestore);
	380
	381	void __lockfunc _read_unlock_irq(rwlock_t *lock)
	382	{
8a25d5de	383	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	384	_raw_read_unlock(lock);
	385	local_irq_enable();
	386	preempt_enable();
	387	}
	388	EXPORT_SYMBOL(_read_unlock_irq);
	389
	390	void __lockfunc _read_unlock_bh(rwlock_t *lock)
	391	{
8a25d5de	392	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	393	_raw_read_unlock(lock);
10f02d1c	394	preempt_enable_no_resched();
8a25d5de	395	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	396	}
	397	EXPORT_SYMBOL(_read_unlock_bh);
	398
	399	void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	400	{
8a25d5de	401	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	402	_raw_write_unlock(lock);
	403	local_irq_restore(flags);
	404	preempt_enable();
	405	}
	406	EXPORT_SYMBOL(_write_unlock_irqrestore);
	407
	408	void __lockfunc _write_unlock_irq(rwlock_t *lock)
	409	{
8a25d5de	410	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	411	_raw_write_unlock(lock);
	412	local_irq_enable();
	413	preempt_enable();
	414	}
	415	EXPORT_SYMBOL(_write_unlock_irq);
	416
	417	void __lockfunc _write_unlock_bh(rwlock_t *lock)
	418	{
8a25d5de	419	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	420	_raw_write_unlock(lock);
10f02d1c	421	preempt_enable_no_resched();
8a25d5de	422	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	423	}
	424	EXPORT_SYMBOL(_write_unlock_bh);
	425
	426	int __lockfunc _spin_trylock_bh(spinlock_t *lock)
	427	{
	428	local_bh_disable();
	429	preempt_disable();
8a25d5de IM	430	if (_raw_spin_trylock(lock)) {
8a25d5de IM	431	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	432	return 1;
8a25d5de	433	}
1da177e4	434
10f02d1c	435	preempt_enable_no_resched();
8a25d5de	436	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	437	return 0;
	438	}
	439	EXPORT_SYMBOL(_spin_trylock_bh);
	440
0764d23c	441	notrace int in_lock_functions(unsigned long addr)
1da177e4 LT	442	{
	443	/* Linker adds these: start and end of __lockfunc functions */
	444	extern char __lock_text_start[], __lock_text_end[];
	445
	446	return addr >= (unsigned long)__lock_text_start
	447	&& addr < (unsigned long)__lock_text_end;
	448	}
	449	EXPORT_SYMBOL(in_lock_functions);