[linux.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

void local_bh_enable(void)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());
#endif
	WARN_ON_ONCE(irqs_disabled());

#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			h->action(h);
			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

EXPORT_SYMBOL(do_softirq);

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(smp_processor_id()))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline fastcall void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

EXPORT_SYMBOL(raise_softirq_irqoff);

void fastcall raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
{
	softirq_vec[nr].data = data;
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *list;
};

/* Some compilers disobey section attribute on statics when not
   initialized -- RR */
static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };

void fastcall __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = __get_cpu_var(tasklet_vec).list;
	__get_cpu_var(tasklet_vec).list = t;
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = __get_cpu_var(tasklet_hi_vec).list;
	__get_cpu_var(tasklet_hi_vec).list = t;
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).list;
	__get_cpu_var(tasklet_vec).list = NULL;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = __get_cpu_var(tasklet_vec).list;
		__get_cpu_var(tasklet_vec).list = t;
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).list;
	__get_cpu_var(tasklet_hi_vec).list = NULL;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = __get_cpu_var(tasklet_hi_vec).list;
		__get_cpu_var(tasklet_hi_vec).list = t;
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
}

static int ksoftirqd(void * __bind_cpu)
{
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	struct tasklet_struct **i;

	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
	*i = per_cpu(tasklet_vec, cpu).list;
	per_cpu(tasklet_vec, cpu).list = NULL;
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
	*i = per_cpu(tasklet_hi_vec, cpu).list;
	per_cpu(tasklet_hi_vec, cpu).list = NULL;
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

__init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, retry, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
	6	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
	7	*/
	8
	9	#include <linux/module.h>
	10	#include <linux/kernel_stat.h>
	11	#include <linux/interrupt.h>
	12	#include <linux/init.h>
	13	#include <linux/mm.h>
	14	#include <linux/notifier.h>
	15	#include <linux/percpu.h>
	16	#include <linux/cpu.h>
83144186	17	#include <linux/freezer.h>
1da177e4 LT	18	#include <linux/kthread.h>
1da177e4 LT	19	#include <linux/rcupdate.h>
78eef01b	20	#include <linux/smp.h>
79bf2bb3	21	#include <linux/tick.h>
1da177e4 LT	22
	23	#include <asm/irq.h>
	24	/*
	25	- No shared variables, all the data are CPU local.
	26	- If a softirq needs serialization, let it serialize itself
	27	by its own spinlocks.
	28	- Even if softirq is serialized, only local cpu is marked for
	29	execution. Hence, we get something sort of weak cpu binding.
	30	Though it is still not clear, will it result in better locality
	31	or will not.
	32
	33	Examples:
	34	- NET RX softirq. It is multithreaded and does not require
	35	any global serialization.
	36	- NET TX softirq. It kicks software netdevice queues, hence
	37	it is logically serialized per device, but this serialization
	38	is invisible to common code.
	39	- Tasklets: serialized wrt itself.
	40	*/
	41
	42	#ifndef __ARCH_IRQ_STAT
	43	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	44	EXPORT_SYMBOL(irq_stat);
	45	#endif
	46
	47	static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
	48
	49	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	50
	51	/*
	52	* we cannot loop indefinitely here to avoid userspace starvation,
	53	* but we also don't want to introduce a worst case 1/HZ latency
	54	* to the pending events, so lets the scheduler to balance
	55	* the softirq load for us.
	56	*/
	57	static inline void wakeup_softirqd(void)
	58	{
	59	/* Interrupts are disabled: no need to stop preemption */
	60	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	61
	62	if (tsk && tsk->state != TASK_RUNNING)
	63	wake_up_process(tsk);
	64	}
	65
de30a2b3 IM	66	/*
	67	* This one is for softirq.c-internal use,
	68	* where hardirqs are disabled legitimately:
	69	*/
3c829c36	70	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	71	static void __local_bh_disable(unsigned long ip)
	72	{
	73	unsigned long flags;
	74
	75	WARN_ON_ONCE(in_irq());
	76
	77	raw_local_irq_save(flags);
	78	add_preempt_count(SOFTIRQ_OFFSET);
	79	/*
	80	* Were softirqs turned off above:
	81	*/
	82	if (softirq_count() == SOFTIRQ_OFFSET)
	83	trace_softirqs_off(ip);
	84	raw_local_irq_restore(flags);
	85	}
3c829c36 TC	86	#else /* !CONFIG_TRACE_IRQFLAGS */
	87	static inline void __local_bh_disable(unsigned long ip)
	88	{
	89	add_preempt_count(SOFTIRQ_OFFSET);
	90	barrier();
	91	}
	92	#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 IM	93
	94	void local_bh_disable(void)
	95	{
	96	__local_bh_disable((unsigned long)__builtin_return_address(0));
	97	}
	98
	99	EXPORT_SYMBOL(local_bh_disable);
	100
	101	void __local_bh_enable(void)
	102	{
	103	WARN_ON_ONCE(in_irq());
	104
	105	/*
	106	* softirqs should never be enabled by __local_bh_enable(),
	107	* it always nests inside local_bh_enable() sections:
	108	*/
	109	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	110
	111	sub_preempt_count(SOFTIRQ_OFFSET);
	112	}
	113	EXPORT_SYMBOL_GPL(__local_bh_enable);
	114
	115	/*
	116	* Special-case - softirqs can safely be enabled in
	117	* cond_resched_softirq(), or by __do_softirq(),
	118	* without processing still-pending softirqs:
	119	*/
	120	void _local_bh_enable(void)
	121	{
	122	WARN_ON_ONCE(in_irq());
	123	WARN_ON_ONCE(!irqs_disabled());
	124
	125	if (softirq_count() == SOFTIRQ_OFFSET)
	126	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	127	sub_preempt_count(SOFTIRQ_OFFSET);
	128	}
	129
	130	EXPORT_SYMBOL(_local_bh_enable);
	131
	132	void local_bh_enable(void)
	133	{
3c829c36	134	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	135	unsigned long flags;
	136
	137	WARN_ON_ONCE(in_irq());
3c829c36	138	#endif
de30a2b3 IM	139	WARN_ON_ONCE(irqs_disabled());
de30a2b3 IM	140
3c829c36	141	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	142	local_irq_save(flags);
3c829c36	143	#endif
de30a2b3 IM	144	/*
	145	* Are softirqs going to be turned on now:
	146	*/
	147	if (softirq_count() == SOFTIRQ_OFFSET)
	148	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	149	/*
	150	* Keep preemption disabled until we are done with
	151	* softirq processing:
	152	*/
	153	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	154
	155	if (unlikely(!in_interrupt() && local_softirq_pending()))
	156	do_softirq();
	157
	158	dec_preempt_count();
3c829c36	159	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	160	local_irq_restore(flags);
3c829c36	161	#endif
de30a2b3 IM	162	preempt_check_resched();
	163	}
	164	EXPORT_SYMBOL(local_bh_enable);
	165
	166	void local_bh_enable_ip(unsigned long ip)
	167	{
3c829c36	168	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	169	unsigned long flags;
	170
	171	WARN_ON_ONCE(in_irq());
	172
	173	local_irq_save(flags);
3c829c36	174	#endif
de30a2b3 IM	175	/*
	176	* Are softirqs going to be turned on now:
	177	*/
	178	if (softirq_count() == SOFTIRQ_OFFSET)
	179	trace_softirqs_on(ip);
	180	/*
	181	* Keep preemption disabled until we are done with
	182	* softirq processing:
	183	*/
	184	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	185
	186	if (unlikely(!in_interrupt() && local_softirq_pending()))
	187	do_softirq();
	188
	189	dec_preempt_count();
3c829c36	190	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	191	local_irq_restore(flags);
3c829c36	192	#endif
de30a2b3 IM	193	preempt_check_resched();
	194	}
	195	EXPORT_SYMBOL(local_bh_enable_ip);
	196
1da177e4 LT	197	/*
	198	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	199	* and we fall back to softirqd after that.
	200	*
	201	* This number has been established via experimentation.
	202	* The two things to balance is latency against fairness -
	203	* we want to handle softirqs as soon as possible, but they
	204	* should not be able to lock up the box.
	205	*/
	206	#define MAX_SOFTIRQ_RESTART 10
	207
	208	asmlinkage void __do_softirq(void)
	209	{
	210	struct softirq_action *h;
	211	__u32 pending;
	212	int max_restart = MAX_SOFTIRQ_RESTART;
	213	int cpu;
	214
	215	pending = local_softirq_pending();
829035fd PM	216	account_system_vtime(current);
829035fd PM	217
de30a2b3 IM	218	__local_bh_disable((unsigned long)__builtin_return_address(0));
de30a2b3 IM	219	trace_softirq_enter();
1da177e4	220
1da177e4 LT	221	cpu = smp_processor_id();
	222	restart:
	223	/* Reset the pending bitmask before enabling irqs */
3f74478b	224	set_softirq_pending(0);
1da177e4	225
c70f5d66	226	local_irq_enable();
1da177e4 LT	227
	228	h = softirq_vec;
	229
	230	do {
	231	if (pending & 1) {
	232	h->action(h);
	233	rcu_bh_qsctr_inc(cpu);
	234	}
	235	h++;
	236	pending >>= 1;
	237	} while (pending);
	238
c70f5d66	239	local_irq_disable();
1da177e4 LT	240
	241	pending = local_softirq_pending();
	242	if (pending && --max_restart)
	243	goto restart;
	244
	245	if (pending)
	246	wakeup_softirqd();
	247
de30a2b3	248	trace_softirq_exit();
829035fd PM	249
829035fd PM	250	account_system_vtime(current);
de30a2b3	251	_local_bh_enable();
1da177e4 LT	252	}
	253
	254	#ifndef __ARCH_HAS_DO_SOFTIRQ
	255
	256	asmlinkage void do_softirq(void)
	257	{
	258	__u32 pending;
	259	unsigned long flags;
	260
	261	if (in_interrupt())
	262	return;
	263
	264	local_irq_save(flags);
	265
	266	pending = local_softirq_pending();
	267
	268	if (pending)
	269	__do_softirq();
	270
	271	local_irq_restore(flags);
	272	}
	273
	274	EXPORT_SYMBOL(do_softirq);
	275
	276	#endif
	277
dde4b2b5 IM	278	/*
	279	* Enter an interrupt context.
	280	*/
	281	void irq_enter(void)
	282	{
79bf2bb3 TG	283	__irq_enter();
	284	#ifdef CONFIG_NO_HZ
	285	if (idle_cpu(smp_processor_id()))
	286	tick_nohz_update_jiffies();
	287	#endif
dde4b2b5 IM	288	}
dde4b2b5 IM	289
1da177e4 LT	290	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	291	# define invoke_softirq() __do_softirq()
	292	#else
	293	# define invoke_softirq() do_softirq()
	294	#endif
	295
	296	/*
	297	* Exit an interrupt context. Process softirqs if needed and possible:
	298	*/
	299	void irq_exit(void)
	300	{
	301	account_system_vtime(current);
de30a2b3	302	trace_hardirq_exit();
1da177e4 LT	303	sub_preempt_count(IRQ_EXIT_OFFSET);
	304	if (!in_interrupt() && local_softirq_pending())
	305	invoke_softirq();
79bf2bb3 TG	306
	307	#ifdef CONFIG_NO_HZ
	308	/* Make sure that timer wheel updates are propagated */
	309	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
	310	tick_nohz_stop_sched_tick();
	311	#endif
1da177e4 LT	312	preempt_enable_no_resched();
	313	}
	314
	315	/*
	316	* This function must run with irqs disabled!
	317	*/
	318	inline fastcall void raise_softirq_irqoff(unsigned int nr)
	319	{
	320	__raise_softirq_irqoff(nr);
	321
	322	/*
	323	* If we're in an interrupt or softirq, we're done
	324	* (this also catches softirq-disabled code). We will
	325	* actually run the softirq once we return from
	326	* the irq or softirq.
	327	*
	328	* Otherwise we wake up ksoftirqd to make sure we
	329	* schedule the softirq soon.
	330	*/
	331	if (!in_interrupt())
	332	wakeup_softirqd();
	333	}
	334
	335	EXPORT_SYMBOL(raise_softirq_irqoff);
	336
	337	void fastcall raise_softirq(unsigned int nr)
	338	{
	339	unsigned long flags;
	340
	341	local_irq_save(flags);
	342	raise_softirq_irqoff(nr);
	343	local_irq_restore(flags);
	344	}
	345
	346	void open_softirq(int nr, void (action)(struct softirq_action), void *data)
	347	{
	348	softirq_vec[nr].data = data;
	349	softirq_vec[nr].action = action;
	350	}
	351
1da177e4 LT	352	/* Tasklets */
	353	struct tasklet_head
	354	{
	355	struct tasklet_struct *list;
	356	};
	357
	358	/* Some compilers disobey section attribute on statics when not
	359	initialized -- RR */
	360	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
	361	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
	362
	363	void fastcall __tasklet_schedule(struct tasklet_struct *t)
	364	{
	365	unsigned long flags;
	366
	367	local_irq_save(flags);
	368	t->next = __get_cpu_var(tasklet_vec).list;
	369	__get_cpu_var(tasklet_vec).list = t;
	370	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	371	local_irq_restore(flags);
	372	}
	373
	374	EXPORT_SYMBOL(__tasklet_schedule);
	375
	376	void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
	377	{
	378	unsigned long flags;
	379
	380	local_irq_save(flags);
	381	t->next = __get_cpu_var(tasklet_hi_vec).list;
	382	__get_cpu_var(tasklet_hi_vec).list = t;
	383	raise_softirq_irqoff(HI_SOFTIRQ);
	384	local_irq_restore(flags);
	385	}
	386
	387	EXPORT_SYMBOL(__tasklet_hi_schedule);
	388
	389	static void tasklet_action(struct softirq_action *a)
	390	{
	391	struct tasklet_struct *list;
	392
	393	local_irq_disable();
	394	list = __get_cpu_var(tasklet_vec).list;
	395	__get_cpu_var(tasklet_vec).list = NULL;
	396	local_irq_enable();
	397
	398	while (list) {
	399	struct tasklet_struct *t = list;
	400
	401	list = list->next;
	402
	403	if (tasklet_trylock(t)) {
	404	if (!atomic_read(&t->count)) {
	405	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	406	BUG();
	407	t->func(t->data);
	408	tasklet_unlock(t);
	409	continue;
	410	}
	411	tasklet_unlock(t);
	412	}
	413
	414	local_irq_disable();
	415	t->next = __get_cpu_var(tasklet_vec).list;
416	__get_cpu_var(tasklet_vec).list = t;
417	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
418	local_irq_enable();
419	}
420	}
421
422	static void tasklet_hi_action(struct softirq_action *a)
423	{
424	struct tasklet_struct *list;
425
426	local_irq_disable();
427	list = __get_cpu_var(tasklet_hi_vec).list;
428	__get_cpu_var(tasklet_hi_vec).list = NULL;
429	local_irq_enable();
430
431	while (list) {
432	struct tasklet_struct *t = list;
433
434	list = list->next;
435
436	if (tasklet_trylock(t)) {
437	if (!atomic_read(&t->count)) {
438	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
439	BUG();
440	t->func(t->data);
441	tasklet_unlock(t);
442	continue;
443	}
444	tasklet_unlock(t);
445	}
446
447	local_irq_disable();
448	t->next = __get_cpu_var(tasklet_hi_vec).list;
449	__get_cpu_var(tasklet_hi_vec).list = t;
450	__raise_softirq_irqoff(HI_SOFTIRQ);
451	local_irq_enable();
452	}
453	}
454
455
456	void tasklet_init(struct tasklet_struct *t,
457	void (*func)(unsigned long), unsigned long data)
458	{
459	t->next = NULL;
460	t->state = 0;
461	atomic_set(&t->count, 0);
462	t->func = func;
463	t->data = data;
464	}
465
466	EXPORT_SYMBOL(tasklet_init);
467
468	void tasklet_kill(struct tasklet_struct *t)
469	{
470	if (in_interrupt())
471	printk("Attempt to kill tasklet from interrupt\n");
472
473	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
474	do
475	yield();
476	while (test_bit(TASKLET_STATE_SCHED, &t->state));
477	}
478	tasklet_unlock_wait(t);
479	clear_bit(TASKLET_STATE_SCHED, &t->state);
480	}
481
482	EXPORT_SYMBOL(tasklet_kill);
483
484	void __init softirq_init(void)
485	{
486	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
487	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
488	}
489
490	static int ksoftirqd(void * __bind_cpu)
491	{
1da177e4 LT	492	set_current_state(TASK_INTERRUPTIBLE);
	493
	494	while (!kthread_should_stop()) {
	495	preempt_disable();
	496	if (!local_softirq_pending()) {
	497	preempt_enable_no_resched();
	498	schedule();
	499	preempt_disable();
	500	}
	501
	502	__set_current_state(TASK_RUNNING);
	503
	504	while (local_softirq_pending()) {
	505	/* Preempt disable stops cpu going offline.
	506	If already offline, we'll be on wrong CPU:
	507	don't process */
	508	if (cpu_is_offline((long)__bind_cpu))
	509	goto wait_to_die;
	510	do_softirq();
	511	preempt_enable_no_resched();
	512	cond_resched();
	513	preempt_disable();
	514	}
	515	preempt_enable();
	516	set_current_state(TASK_INTERRUPTIBLE);
	517	}
	518	__set_current_state(TASK_RUNNING);
	519	return 0;
	520
	521	wait_to_die:
	522	preempt_enable();
	523	/* Wait for kthread_stop */
	524	set_current_state(TASK_INTERRUPTIBLE);
	525	while (!kthread_should_stop()) {
	526	schedule();
	527	set_current_state(TASK_INTERRUPTIBLE);
	528	}
	529	__set_current_state(TASK_RUNNING);
	530	return 0;
	531	}
	532
	533	#ifdef CONFIG_HOTPLUG_CPU
	534	/*
	535	* tasklet_kill_immediate is called to remove a tasklet which can already be
	536	* scheduled for execution on @cpu.
	537	*
	538	* Unlike tasklet_kill, this function removes the tasklet
	539	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	540	*
	541	* When this function is called, @cpu must be in the CPU_DEAD state.
	542	*/
	543	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	544	{
	545	struct tasklet_struct **i;
	546
	547	BUG_ON(cpu_online(cpu));
	548	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	549
	550	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	551	return;
	552
	553	/* CPU is dead, so no lock needed. */
	554	for (i = &per_cpu(tasklet_vec, cpu).list; i; i = &(i)->next) {
	555	if (*i == t) {
556	*i = t->next;
557	return;
558	}
559	}
560	BUG();
561	}
562
563	static void takeover_tasklets(unsigned int cpu)
564	{
565	struct tasklet_struct **i;
566
567	/* CPU is dead, so no lock needed. */
568	local_irq_disable();
569
570	/* Find end, append list for that CPU. */
571	for (i = &__get_cpu_var(tasklet_vec).list; i; i = &(i)->next);
572	*i = per_cpu(tasklet_vec, cpu).list;
573	per_cpu(tasklet_vec, cpu).list = NULL;
574	raise_softirq_irqoff(TASKLET_SOFTIRQ);
575
576	for (i = &__get_cpu_var(tasklet_hi_vec).list; i; i = &(i)->next);
577	*i = per_cpu(tasklet_hi_vec, cpu).list;
578	per_cpu(tasklet_hi_vec, cpu).list = NULL;
579	raise_softirq_irqoff(HI_SOFTIRQ);
580
581	local_irq_enable();
582	}
583	#endif /* CONFIG_HOTPLUG_CPU */
584
8c78f307	585	static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4 LT	586	unsigned long action,
	587	void *hcpu)
	588	{
	589	int hotcpu = (unsigned long)hcpu;
	590	struct task_struct *p;
	591
	592	switch (action) {
	593	case CPU_UP_PREPARE:
8bb78442	594	case CPU_UP_PREPARE_FROZEN:
1da177e4 LT	595	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	596	if (IS_ERR(p)) {
	597	printk("ksoftirqd for %i failed\n", hotcpu);
	598	return NOTIFY_BAD;
	599	}
	600	kthread_bind(p, hotcpu);
	601	per_cpu(ksoftirqd, hotcpu) = p;
	602	break;
	603	case CPU_ONLINE:
8bb78442	604	case CPU_ONLINE_FROZEN:
1da177e4 LT	605	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	606	break;
	607	#ifdef CONFIG_HOTPLUG_CPU
	608	case CPU_UP_CANCELED:
8bb78442	609	case CPU_UP_CANCELED_FROZEN:
fc75cdfa HC	610	if (!per_cpu(ksoftirqd, hotcpu))
fc75cdfa HC	611	break;
1da177e4	612	/* Unbind so it can run. Fall thru. */
a4c4af7c HC	613	kthread_bind(per_cpu(ksoftirqd, hotcpu),
a4c4af7c HC	614	any_online_cpu(cpu_online_map));
1da177e4	615	case CPU_DEAD:
1c6b4aa9 ST	616	case CPU_DEAD_FROZEN: {
	617	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	618
1da177e4 LT	619	p = per_cpu(ksoftirqd, hotcpu);
1da177e4 LT	620	per_cpu(ksoftirqd, hotcpu) = NULL;
1c6b4aa9	621	sched_setscheduler(p, SCHED_FIFO, &param);
1da177e4 LT	622	kthread_stop(p);
	623	takeover_tasklets(hotcpu);
	624	break;
1c6b4aa9	625	}
1da177e4 LT	626	#endif /* CONFIG_HOTPLUG_CPU */
	627	}
	628	return NOTIFY_OK;
	629	}
	630
8c78f307	631	static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4 LT	632	.notifier_call = cpu_callback
	633	};
	634
	635	__init int spawn_ksoftirqd(void)
	636	{
	637	void cpu = (void )(long)smp_processor_id();
07dccf33 AM	638	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	639
	640	BUG_ON(err == NOTIFY_BAD);
1da177e4 LT	641	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	642	register_cpu_notifier(&cpu_nfb);
	643	return 0;
	644	}
78eef01b AM	645
	646	#ifdef CONFIG_SMP
	647	/*
	648	* Call a function on all processors
	649	*/
	650	int on_each_cpu(void (func) (void info), void *info, int retry, int wait)
	651	{
	652	int ret = 0;
	653
	654	preempt_disable();
	655	ret = smp_call_function(func, info, retry, wait);
	656	local_irq_disable();
	657	func(info);
	658	local_irq_enable();
	659	preempt_enable();
	660	return ret;
	661	}
	662	EXPORT_SYMBOL(on_each_cpu);
	663	#endif