[linux.git] / kernel / stop_machine.c

/*
 * kernel/stop_machine.c
 *
 * Copyright (C) 2008, 2005	IBM Corporation.
 * Copyright (C) 2008, 2005	Rusty Russell [email protected]
 * Copyright (C) 2010		SUSE Linux Products GmbH
 * Copyright (C) 2010		Tejun Heo <[email protected]>
 *
 * This file is released under the GPLv2 and any later version.
 */
#include <linux/completion.h>
#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/stop_machine.h>
#include <linux/interrupt.h>
#include <linux/kallsyms.h>

#include <asm/atomic.h>

/*
 * Structure to determine completion condition and record errors.  May
 * be shared by works on different cpus.
 */
struct cpu_stop_done {
	atomic_t		nr_todo;	/* nr left to execute */
	bool			executed;	/* actually executed? */
	int			ret;		/* collected return value */
	struct completion	completion;	/* fired if nr_todo reaches 0 */
};

/* the actual stopper, one per every possible cpu, enabled on online cpus */
struct cpu_stopper {
	spinlock_t		lock;
	bool			enabled;	/* is this stopper enabled? */
	struct list_head	works;		/* list of pending works */
	struct task_struct	*thread;	/* stopper thread */
};

static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);

static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
{
	memset(done, 0, sizeof(*done));
	atomic_set(&done->nr_todo, nr_todo);
	init_completion(&done->completion);
}

/* signal completion unless @done is NULL */
static void cpu_stop_signal_done(struct cpu_stop_done *done, bool executed)
{
	if (done) {
		if (executed)
			done->executed = true;
		if (atomic_dec_and_test(&done->nr_todo))
			complete(&done->completion);
	}
}

/* queue @work to @stopper.  if offline, @work is completed immediately */
static void cpu_stop_queue_work(struct cpu_stopper *stopper,
				struct cpu_stop_work *work)
{
	unsigned long flags;

	spin_lock_irqsave(&stopper->lock, flags);

	if (stopper->enabled) {
		list_add_tail(&work->list, &stopper->works);
		wake_up_process(stopper->thread);
	} else
		cpu_stop_signal_done(work->done, false);

	spin_unlock_irqrestore(&stopper->lock, flags);
}

/**
 * stop_one_cpu - stop a cpu
 * @cpu: cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Execute @fn(@arg) on @cpu.  @fn is run in a process context with
 * the highest priority preempting any task on the cpu and
 * monopolizing it.  This function returns after the execution is
 * complete.
 *
 * This function doesn't guarantee @cpu stays online till @fn
 * completes.  If @cpu goes down in the middle, execution may happen
 * partially or fully on different cpus.  @fn should either be ready
 * for that or the caller should ensure that @cpu stays online until
 * this function completes.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
 * otherwise, the return value of @fn.
 */
int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
{
	struct cpu_stop_done done;
	struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };

	cpu_stop_init_done(&done, 1);
	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), &work);
	wait_for_completion(&done.completion);
	return done.executed ? done.ret : -ENOENT;
}

/**
 * stop_one_cpu_nowait - stop a cpu but don't wait for completion
 * @cpu: cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Similar to stop_one_cpu() but doesn't wait for completion.  The
 * caller is responsible for ensuring @work_buf is currently unused
 * and will remain untouched until stopper starts executing @fn.
 *
 * CONTEXT:
 * Don't care.
 */
void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
			struct cpu_stop_work *work_buf)
{
	*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), work_buf);
}

DEFINE_MUTEX(stop_cpus_mutex);
/* static data for stop_cpus */
static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work);

int __stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
{
	struct cpu_stop_work *work;
	struct cpu_stop_done done;
	unsigned int cpu;

	/* initialize works and done */
	for_each_cpu(cpu, cpumask) {
		work = &per_cpu(stop_cpus_work, cpu);
		work->fn = fn;
		work->arg = arg;
		work->done = &done;
	}
	cpu_stop_init_done(&done, cpumask_weight(cpumask));

	/*
	 * Disable preemption while queueing to avoid getting
	 * preempted by a stopper which might wait for other stoppers
	 * to enter @fn which can lead to deadlock.
	 */
	preempt_disable();
	for_each_cpu(cpu, cpumask)
		cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu),
				    &per_cpu(stop_cpus_work, cpu));
	preempt_enable();

	wait_for_completion(&done.completion);
	return done.executed ? done.ret : -ENOENT;
}

/**
 * stop_cpus - stop multiple cpus
 * @cpumask: cpus to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Execute @fn(@arg) on online cpus in @cpumask.  On each target cpu,
 * @fn is run in a process context with the highest priority
 * preempting any task on the cpu and monopolizing it.  This function
 * returns after all executions are complete.
 *
 * This function doesn't guarantee the cpus in @cpumask stay online
 * till @fn completes.  If some cpus go down in the middle, execution
 * on the cpu may happen partially or fully on different cpus.  @fn
 * should either be ready for that or the caller should ensure that
 * the cpus stay online until this function completes.
 *
 * All stop_cpus() calls are serialized making it safe for @fn to wait
 * for all cpus to start executing it.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -ENOENT if @fn(@arg) was not executed at all because all cpus in
 * @cpumask were offline; otherwise, 0 if all executions of @fn
 * returned 0, any non zero return value if any returned non zero.
 */
int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
{
	int ret;

	/* static works are used, process one request at a time */
	mutex_lock(&stop_cpus_mutex);
	ret = __stop_cpus(cpumask, fn, arg);
	mutex_unlock(&stop_cpus_mutex);
	return ret;
}

/**
 * try_stop_cpus - try to stop multiple cpus
 * @cpumask: cpus to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Identical to stop_cpus() except that it fails with -EAGAIN if
 * someone else is already using the facility.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -EAGAIN if someone else is already stopping cpus, -ENOENT if
 * @fn(@arg) was not executed at all because all cpus in @cpumask were
 * offline; otherwise, 0 if all executions of @fn returned 0, any non
 * zero return value if any returned non zero.
 */
int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
{
	int ret;

	/* static works are used, process one request at a time */
	if (!mutex_trylock(&stop_cpus_mutex))
		return -EAGAIN;
	ret = __stop_cpus(cpumask, fn, arg);
	mutex_unlock(&stop_cpus_mutex);
	return ret;
}

static int cpu_stopper_thread(void *data)
{
	struct cpu_stopper *stopper = data;
	struct cpu_stop_work *work;
	int ret;

repeat:
	set_current_state(TASK_INTERRUPTIBLE);	/* mb paired w/ kthread_stop */

	if (kthread_should_stop()) {
		__set_current_state(TASK_RUNNING);
		return 0;
	}

	work = NULL;
	spin_lock_irq(&stopper->lock);
	if (!list_empty(&stopper->works)) {
		work = list_first_entry(&stopper->works,
					struct cpu_stop_work, list);
		list_del_init(&work->list);
	}
	spin_unlock_irq(&stopper->lock);

	if (work) {
		cpu_stop_fn_t fn = work->fn;
		void *arg = work->arg;
		struct cpu_stop_done *done = work->done;
		char ksym_buf[KSYM_NAME_LEN] __maybe_unused;

		__set_current_state(TASK_RUNNING);

		/* cpu stop callbacks are not allowed to sleep */
		preempt_disable();

		ret = fn(arg);
		if (ret)
			done->ret = ret;

		/* restore preemption and check it's still balanced */
		preempt_enable();
		WARN_ONCE(preempt_count(),
			  "cpu_stop: %s(%p) leaked preempt count\n",
			  kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL,
					  ksym_buf), arg);

		cpu_stop_signal_done(done, true);
	} else
		schedule();

	goto repeat;
}

extern void sched_set_stop_task(int cpu, struct task_struct *stop);

/* manage stopper for a cpu, mostly lifted from sched migration thread mgmt */
static int __cpuinit cpu_stop_cpu_callback(struct notifier_block *nfb,
					   unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
	struct task_struct *p;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_UP_PREPARE:
		BUG_ON(stopper->thread || stopper->enabled ||
		       !list_empty(&stopper->works));
		p = kthread_create_on_node(cpu_stopper_thread,
					   stopper,
					   cpu_to_node(cpu),
					   "migration/%d", cpu);
		if (IS_ERR(p))
			return notifier_from_errno(PTR_ERR(p));
		get_task_struct(p);
		kthread_bind(p, cpu);
		sched_set_stop_task(cpu, p);
		stopper->thread = p;
		break;

	case CPU_ONLINE:
		/* strictly unnecessary, as first user will wake it */
		wake_up_process(stopper->thread);
		/* mark enabled */
		spin_lock_irq(&stopper->lock);
		stopper->enabled = true;
		spin_unlock_irq(&stopper->lock);
		break;

#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_POST_DEAD:
	{
		struct cpu_stop_work *work;

		sched_set_stop_task(cpu, NULL);
		/* kill the stopper */
		kthread_stop(stopper->thread);
		/* drain remaining works */
		spin_lock_irq(&stopper->lock);
		list_for_each_entry(work, &stopper->works, list)
			cpu_stop_signal_done(work->done, false);
		stopper->enabled = false;
		spin_unlock_irq(&stopper->lock);
		/* release the stopper */
		put_task_struct(stopper->thread);
		stopper->thread = NULL;
		break;
	}
#endif
	}

	return NOTIFY_OK;
}

/*
 * Give it a higher priority so that cpu stopper is available to other
 * cpu notifiers.  It currently shares the same priority as sched
 * migration_notifier.
 */
static struct notifier_block __cpuinitdata cpu_stop_cpu_notifier = {
	.notifier_call	= cpu_stop_cpu_callback,
	.priority	= 10,
};

static int __init cpu_stop_init(void)
{
	void *bcpu = (void *)(long)smp_processor_id();
	unsigned int cpu;
	int err;

	for_each_possible_cpu(cpu) {
		struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

		spin_lock_init(&stopper->lock);
		INIT_LIST_HEAD(&stopper->works);
	}

	/* start one for the boot cpu */
	err = cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_UP_PREPARE,
				    bcpu);
	BUG_ON(err != NOTIFY_OK);
	cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_ONLINE, bcpu);
	register_cpu_notifier(&cpu_stop_cpu_notifier);

	return 0;
}
early_initcall(cpu_stop_init);

#ifdef CONFIG_STOP_MACHINE

/* This controls the threads on each CPU. */
enum stopmachine_state {
	/* Dummy starting state for thread. */
	STOPMACHINE_NONE,
	/* Awaiting everyone to be scheduled. */
	STOPMACHINE_PREPARE,
	/* Disable interrupts. */
	STOPMACHINE_DISABLE_IRQ,
	/* Run the function */
	STOPMACHINE_RUN,
	/* Exit */
	STOPMACHINE_EXIT,
};

struct stop_machine_data {
	int			(*fn)(void *);
	void			*data;
	/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
	unsigned int		num_threads;
	const struct cpumask	*active_cpus;

	enum stopmachine_state	state;
	atomic_t		thread_ack;
};

static void set_state(struct stop_machine_data *smdata,
		      enum stopmachine_state newstate)
{
	/* Reset ack counter. */
	atomic_set(&smdata->thread_ack, smdata->num_threads);
	smp_wmb();
	smdata->state = newstate;
}

/* Last one to ack a state moves to the next state. */
static void ack_state(struct stop_machine_data *smdata)
{
	if (atomic_dec_and_test(&smdata->thread_ack))
		set_state(smdata, smdata->state + 1);
}

/* This is the cpu_stop function which stops the CPU. */
static int stop_machine_cpu_stop(void *data)
{
	struct stop_machine_data *smdata = data;
	enum stopmachine_state curstate = STOPMACHINE_NONE;
	int cpu = smp_processor_id(), err = 0;
	bool is_active;

	if (!smdata->active_cpus)
		is_active = cpu == cpumask_first(cpu_online_mask);
	else
		is_active = cpumask_test_cpu(cpu, smdata->active_cpus);

	/* Simple state machine */
	do {
		/* Chill out and ensure we re-read stopmachine_state. */
		cpu_relax();
		if (smdata->state != curstate) {
			curstate = smdata->state;
			switch (curstate) {
			case STOPMACHINE_DISABLE_IRQ:
				local_irq_disable();
				hard_irq_disable();
				break;
			case STOPMACHINE_RUN:
				if (is_active)
					err = smdata->fn(smdata->data);
				break;
			default:
				break;
			}
			ack_state(smdata);
		}
	} while (curstate != STOPMACHINE_EXIT);

	local_irq_enable();
	return err;
}

int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
{
	struct stop_machine_data smdata = { .fn = fn, .data = data,
					    .num_threads = num_online_cpus(),
					    .active_cpus = cpus };

	/* Set the initial state and stop all online cpus. */
	set_state(&smdata, STOPMACHINE_PREPARE);
	return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata);
}

int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
{
	int ret;

	/* No CPUs can come up or down during this. */
	get_online_cpus();
	ret = __stop_machine(fn, data, cpus);
	put_online_cpus();
	return ret;
}
EXPORT_SYMBOL_GPL(stop_machine);

#endif	/* CONFIG_STOP_MACHINE */
Commit	Line	Data
1142d810 TH	1	/*
	2	* kernel/stop_machine.c
	3	*
	4	* Copyright (C) 2008, 2005 IBM Corporation.
	5	* Copyright (C) 2008, 2005 Rusty Russell [email protected]
	6	* Copyright (C) 2010 SUSE Linux Products GmbH
	7	* Copyright (C) 2010 Tejun Heo <[email protected]>
	8	*
	9	* This file is released under the GPLv2 and any later version.
e5582ca2	10	*/
1142d810	11	#include <linux/completion.h>
1da177e4	12	#include <linux/cpu.h>
1142d810	13	#include <linux/init.h>
ee527cd3 PB	14	#include <linux/kthread.h>
ee527cd3 PB	15	#include <linux/module.h>
1142d810	16	#include <linux/percpu.h>
ee527cd3 PB	17	#include <linux/sched.h>
ee527cd3 PB	18	#include <linux/stop_machine.h>
a12bb444	19	#include <linux/interrupt.h>
1142d810	20	#include <linux/kallsyms.h>
a12bb444	21
1da177e4	22	#include <asm/atomic.h>
1142d810 TH	23
	24	/*
	25	* Structure to determine completion condition and record errors. May
	26	* be shared by works on different cpus.
	27	*/
	28	struct cpu_stop_done {
	29	atomic_t nr_todo; /* nr left to execute */
	30	bool executed; /* actually executed? */
	31	int ret; /* collected return value */
	32	struct completion completion; /* fired if nr_todo reaches 0 */
	33	};
	34
	35	/* the actual stopper, one per every possible cpu, enabled on online cpus */
	36	struct cpu_stopper {
	37	spinlock_t lock;
878ae127	38	bool enabled; /* is this stopper enabled? */
1142d810 TH	39	struct list_head works; /* list of pending works */
1142d810 TH	40	struct task_struct thread; / stopper thread */
1142d810 TH	41	};
	42
	43	static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
	44
	45	static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
	46	{
	47	memset(done, 0, sizeof(*done));
	48	atomic_set(&done->nr_todo, nr_todo);
	49	init_completion(&done->completion);
	50	}
	51
	52	/* signal completion unless @done is NULL */
	53	static void cpu_stop_signal_done(struct cpu_stop_done *done, bool executed)
	54	{
	55	if (done) {
	56	if (executed)
	57	done->executed = true;
	58	if (atomic_dec_and_test(&done->nr_todo))
	59	complete(&done->completion);
	60	}
	61	}
	62
	63	/* queue @work to @stopper. if offline, @work is completed immediately */
	64	static void cpu_stop_queue_work(struct cpu_stopper *stopper,
	65	struct cpu_stop_work *work)
	66	{
	67	unsigned long flags;
	68
	69	spin_lock_irqsave(&stopper->lock, flags);
	70
	71	if (stopper->enabled) {
	72	list_add_tail(&work->list, &stopper->works);
	73	wake_up_process(stopper->thread);
	74	} else
	75	cpu_stop_signal_done(work->done, false);
	76
	77	spin_unlock_irqrestore(&stopper->lock, flags);
	78	}
	79
	80	/**
	81	* stop_one_cpu - stop a cpu
	82	* @cpu: cpu to stop
	83	* @fn: function to execute
	84	* @arg: argument to @fn
	85	*
	86	* Execute @fn(@arg) on @cpu. @fn is run in a process context with
	87	* the highest priority preempting any task on the cpu and
	88	* monopolizing it. This function returns after the execution is
	89	* complete.
	90	*
	91	* This function doesn't guarantee @cpu stays online till @fn
	92	* completes. If @cpu goes down in the middle, execution may happen
	93	* partially or fully on different cpus. @fn should either be ready
	94	* for that or the caller should ensure that @cpu stays online until
	95	* this function completes.
	96	*
	97	* CONTEXT:
	98	* Might sleep.
	99	*
	100	* RETURNS:
	101	* -ENOENT if @fn(@arg) was not executed because @cpu was offline;
	102	* otherwise, the return value of @fn.
	103	*/
	104	int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
105	{
106	struct cpu_stop_done done;
107	struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
108
109	cpu_stop_init_done(&done, 1);
110	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), &work);
111	wait_for_completion(&done.completion);
112	return done.executed ? done.ret : -ENOENT;
113	}
114
115	/**
116	* stop_one_cpu_nowait - stop a cpu but don't wait for completion
117	* @cpu: cpu to stop
118	* @fn: function to execute
119	* @arg: argument to @fn
120	*
121	* Similar to stop_one_cpu() but doesn't wait for completion. The
122	* caller is responsible for ensuring @work_buf is currently unused
123	* and will remain untouched until stopper starts executing @fn.
124	*
125	* CONTEXT:
126	* Don't care.
127	*/
128	void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
129	struct cpu_stop_work *work_buf)
130	{
131	*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
132	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), work_buf);
133	}
134
6d3321e8	135	DEFINE_MUTEX(stop_cpus_mutex);
1142d810	136	/* static data for stop_cpus */
1142d810 TH	137	static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work);
	138
	139	int __stop_cpus(const struct cpumask cpumask, cpu_stop_fn_t fn, void arg)
	140	{
	141	struct cpu_stop_work *work;
	142	struct cpu_stop_done done;
	143	unsigned int cpu;
	144
	145	/* initialize works and done */
	146	for_each_cpu(cpu, cpumask) {
	147	work = &per_cpu(stop_cpus_work, cpu);
	148	work->fn = fn;
	149	work->arg = arg;
	150	work->done = &done;
	151	}
	152	cpu_stop_init_done(&done, cpumask_weight(cpumask));
	153
	154	/*
	155	* Disable preemption while queueing to avoid getting
	156	* preempted by a stopper which might wait for other stoppers
	157	* to enter @fn which can lead to deadlock.
	158	*/
	159	preempt_disable();
	160	for_each_cpu(cpu, cpumask)
	161	cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu),
	162	&per_cpu(stop_cpus_work, cpu));
	163	preempt_enable();
	164
	165	wait_for_completion(&done.completion);
	166	return done.executed ? done.ret : -ENOENT;
	167	}
	168
	169	/**
	170	* stop_cpus - stop multiple cpus
	171	* @cpumask: cpus to stop
	172	* @fn: function to execute
	173	* @arg: argument to @fn
	174	*
	175	* Execute @fn(@arg) on online cpus in @cpumask. On each target cpu,
	176	* @fn is run in a process context with the highest priority
	177	* preempting any task on the cpu and monopolizing it. This function
	178	* returns after all executions are complete.
	179	*
	180	* This function doesn't guarantee the cpus in @cpumask stay online
	181	* till @fn completes. If some cpus go down in the middle, execution
	182	* on the cpu may happen partially or fully on different cpus. @fn
	183	* should either be ready for that or the caller should ensure that
	184	* the cpus stay online until this function completes.
	185	*
	186	* All stop_cpus() calls are serialized making it safe for @fn to wait
	187	* for all cpus to start executing it.
	188	*
	189	* CONTEXT:
	190	* Might sleep.
	191	*
	192	* RETURNS:
	193	* -ENOENT if @fn(@arg) was not executed at all because all cpus in
	194	* @cpumask were offline; otherwise, 0 if all executions of @fn
	195	* returned 0, any non zero return value if any returned non zero.
	196	*/
	197	int stop_cpus(const struct cpumask cpumask, cpu_stop_fn_t fn, void arg)
	198	{
	199	int ret;
	200
201	/* static works are used, process one request at a time */
202	mutex_lock(&stop_cpus_mutex);
203	ret = __stop_cpus(cpumask, fn, arg);
204	mutex_unlock(&stop_cpus_mutex);
205	return ret;
206	}
207
208	/**
209	* try_stop_cpus - try to stop multiple cpus
210	* @cpumask: cpus to stop
211	* @fn: function to execute
212	* @arg: argument to @fn
213	*
214	* Identical to stop_cpus() except that it fails with -EAGAIN if
215	* someone else is already using the facility.
216	*
217	* CONTEXT:
218	* Might sleep.
219	*
220	* RETURNS:
221	* -EAGAIN if someone else is already stopping cpus, -ENOENT if
222	* @fn(@arg) was not executed at all because all cpus in @cpumask were
223	* offline; otherwise, 0 if all executions of @fn returned 0, any non
224	* zero return value if any returned non zero.
225	*/
226	int try_stop_cpus(const struct cpumask cpumask, cpu_stop_fn_t fn, void arg)
227	{
228	int ret;
229
230	/* static works are used, process one request at a time */
231	if (!mutex_trylock(&stop_cpus_mutex))
232	return -EAGAIN;
233	ret = __stop_cpus(cpumask, fn, arg);
234	mutex_unlock(&stop_cpus_mutex);
235	return ret;
236	}
237
238	static int cpu_stopper_thread(void *data)
239	{
240	struct cpu_stopper *stopper = data;
241	struct cpu_stop_work *work;
242	int ret;
243
244	repeat:
245	set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */
246
247	if (kthread_should_stop()) {
248	__set_current_state(TASK_RUNNING);
249	return 0;
250	}
251
252	work = NULL;
253	spin_lock_irq(&stopper->lock);
254	if (!list_empty(&stopper->works)) {
255	work = list_first_entry(&stopper->works,
256	struct cpu_stop_work, list);
257	list_del_init(&work->list);
258	}
259	spin_unlock_irq(&stopper->lock);
260
261	if (work) {
262	cpu_stop_fn_t fn = work->fn;
263	void *arg = work->arg;
264	struct cpu_stop_done *done = work->done;
ca51c5a7	265	char ksym_buf[KSYM_NAME_LEN] __maybe_unused;
1142d810 TH	266
	267	__set_current_state(TASK_RUNNING);
	268
	269	/* cpu stop callbacks are not allowed to sleep */
	270	preempt_disable();
	271
	272	ret = fn(arg);
	273	if (ret)
	274	done->ret = ret;
	275
	276	/* restore preemption and check it's still balanced */
	277	preempt_enable();
	278	WARN_ONCE(preempt_count(),
	279	"cpu_stop: %s(%p) leaked preempt count\n",
	280	kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL,
	281	ksym_buf), arg);
	282
	283	cpu_stop_signal_done(done, true);
	284	} else
	285	schedule();
	286
	287	goto repeat;
	288	}
	289
34f971f6 PZ	290	extern void sched_set_stop_task(int cpu, struct task_struct *stop);
34f971f6 PZ	291
1142d810 TH	292	/* manage stopper for a cpu, mostly lifted from sched migration thread mgmt */
	293	static int __cpuinit cpu_stop_cpu_callback(struct notifier_block *nfb,
	294	unsigned long action, void *hcpu)
	295	{
1142d810 TH	296	unsigned int cpu = (unsigned long)hcpu;
1142d810 TH	297	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
1142d810 TH	298	struct task_struct *p;
	299
	300	switch (action & ~CPU_TASKS_FROZEN) {
	301	case CPU_UP_PREPARE:
	302	BUG_ON(stopper->thread \|\| stopper->enabled \|\|
	303	!list_empty(&stopper->works));
94dcf29a ED	304	p = kthread_create_on_node(cpu_stopper_thread,
	305	stopper,
	306	cpu_to_node(cpu),
	307	"migration/%d", cpu);
1142d810	308	if (IS_ERR(p))
4ce6494d	309	return notifier_from_errno(PTR_ERR(p));
1142d810	310	get_task_struct(p);
34f971f6 PZ	311	kthread_bind(p, cpu);
34f971f6 PZ	312	sched_set_stop_task(cpu, p);
1142d810 TH	313	stopper->thread = p;
	314	break;
	315
	316	case CPU_ONLINE:
1142d810 TH	317	/* strictly unnecessary, as first user will wake it */
	318	wake_up_process(stopper->thread);
	319	/* mark enabled */
	320	spin_lock_irq(&stopper->lock);
	321	stopper->enabled = true;
	322	spin_unlock_irq(&stopper->lock);
	323	break;
	324
	325	#ifdef CONFIG_HOTPLUG_CPU
	326	case CPU_UP_CANCELED:
54e88fad	327	case CPU_POST_DEAD:
9c6f7e43 IM	328	{
	329	struct cpu_stop_work *work;
	330
34f971f6	331	sched_set_stop_task(cpu, NULL);
1142d810 TH	332	/* kill the stopper */
	333	kthread_stop(stopper->thread);
	334	/* drain remaining works */
	335	spin_lock_irq(&stopper->lock);
	336	list_for_each_entry(work, &stopper->works, list)
	337	cpu_stop_signal_done(work->done, false);
	338	stopper->enabled = false;
	339	spin_unlock_irq(&stopper->lock);
	340	/* release the stopper */
	341	put_task_struct(stopper->thread);
	342	stopper->thread = NULL;
	343	break;
9c6f7e43	344	}
1142d810 TH	345	#endif
	346	}
	347
	348	return NOTIFY_OK;
	349	}
	350
	351	/*
	352	* Give it a higher priority so that cpu stopper is available to other
	353	* cpu notifiers. It currently shares the same priority as sched
	354	* migration_notifier.
	355	*/
	356	static struct notifier_block __cpuinitdata cpu_stop_cpu_notifier = {
	357	.notifier_call = cpu_stop_cpu_callback,
	358	.priority = 10,
	359	};
	360
	361	static int __init cpu_stop_init(void)
	362	{
	363	void bcpu = (void )(long)smp_processor_id();
	364	unsigned int cpu;
	365	int err;
	366
	367	for_each_possible_cpu(cpu) {
	368	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
	369
	370	spin_lock_init(&stopper->lock);
	371	INIT_LIST_HEAD(&stopper->works);
	372	}
	373
	374	/* start one for the boot cpu */
	375	err = cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_UP_PREPARE,
	376	bcpu);
4ce6494d	377	BUG_ON(err != NOTIFY_OK);
1142d810 TH	378	cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_ONLINE, bcpu);
	379	register_cpu_notifier(&cpu_stop_cpu_notifier);
	380
	381	return 0;
	382	}
	383	early_initcall(cpu_stop_init);
1da177e4	384
bbf1bb3e TH	385	#ifdef CONFIG_STOP_MACHINE
bbf1bb3e TH	386
ffdb5976	387	/* This controls the threads on each CPU. */
1da177e4	388	enum stopmachine_state {
ffdb5976 RR	389	/* Dummy starting state for thread. */
	390	STOPMACHINE_NONE,
	391	/* Awaiting everyone to be scheduled. */
1da177e4	392	STOPMACHINE_PREPARE,
ffdb5976	393	/* Disable interrupts. */
1da177e4	394	STOPMACHINE_DISABLE_IRQ,
ffdb5976	395	/* Run the function */
5c2aed62	396	STOPMACHINE_RUN,
ffdb5976	397	/* Exit */
1da177e4 LT	398	STOPMACHINE_EXIT,
	399	};
	400
5c2aed62	401	struct stop_machine_data {
3fc1f1e2 TH	402	int (fn)(void );
	403	void *data;
	404	/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
	405	unsigned int num_threads;
	406	const struct cpumask *active_cpus;
	407
	408	enum stopmachine_state state;
	409	atomic_t thread_ack;
ffdb5976	410	};
5c2aed62	411
3fc1f1e2 TH	412	static void set_state(struct stop_machine_data *smdata,
3fc1f1e2 TH	413	enum stopmachine_state newstate)
1da177e4	414	{
ffdb5976	415	/* Reset ack counter. */
3fc1f1e2	416	atomic_set(&smdata->thread_ack, smdata->num_threads);
ffdb5976	417	smp_wmb();
3fc1f1e2	418	smdata->state = newstate;
1da177e4 LT	419	}
1da177e4 LT	420
ffdb5976	421	/* Last one to ack a state moves to the next state. */
3fc1f1e2	422	static void ack_state(struct stop_machine_data *smdata)
1da177e4	423	{
3fc1f1e2 TH	424	if (atomic_dec_and_test(&smdata->thread_ack))
3fc1f1e2 TH	425	set_state(smdata, smdata->state + 1);
1da177e4 LT	426	}
1da177e4 LT	427
3fc1f1e2 TH	428	/* This is the cpu_stop function which stops the CPU. */
3fc1f1e2 TH	429	static int stop_machine_cpu_stop(void *data)
1da177e4	430	{
3fc1f1e2	431	struct stop_machine_data *smdata = data;
ffdb5976	432	enum stopmachine_state curstate = STOPMACHINE_NONE;
3fc1f1e2 TH	433	int cpu = smp_processor_id(), err = 0;
	434	bool is_active;
	435
	436	if (!smdata->active_cpus)
	437	is_active = cpu == cpumask_first(cpu_online_mask);
	438	else
	439	is_active = cpumask_test_cpu(cpu, smdata->active_cpus);
c9583e55	440
ffdb5976 RR	441	/* Simple state machine */
	442	do {
	443	/* Chill out and ensure we re-read stopmachine_state. */
3401a61e	444	cpu_relax();
3fc1f1e2 TH	445	if (smdata->state != curstate) {
3fc1f1e2 TH	446	curstate = smdata->state;
ffdb5976 RR	447	switch (curstate) {
	448	case STOPMACHINE_DISABLE_IRQ:
	449	local_irq_disable();
	450	hard_irq_disable();
	451	break;
	452	case STOPMACHINE_RUN:
3fc1f1e2 TH	453	if (is_active)
3fc1f1e2 TH	454	err = smdata->fn(smdata->data);
ffdb5976 RR	455	break;
	456	default:
	457	break;
	458	}
3fc1f1e2	459	ack_state(smdata);
ffdb5976 RR	460	}
ffdb5976 RR	461	} while (curstate != STOPMACHINE_EXIT);
1da177e4	462
1da177e4	463	local_irq_enable();
3fc1f1e2	464	return err;
1da177e4 LT	465	}
1da177e4 LT	466
41c7bb95	467	int __stop_machine(int (fn)(void ), void data, const struct cpumask cpus)
1da177e4	468	{
3fc1f1e2 TH	469	struct stop_machine_data smdata = { .fn = fn, .data = data,
	470	.num_threads = num_online_cpus(),
	471	.active_cpus = cpus };
	472
	473	/* Set the initial state and stop all online cpus. */
	474	set_state(&smdata, STOPMACHINE_PREPARE);
	475	return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata);
1da177e4 LT	476	}
1da177e4 LT	477
41c7bb95	478	int stop_machine(int (fn)(void ), void data, const struct cpumask cpus)
1da177e4	479	{
1da177e4 LT	480	int ret;
	481
	482	/* No CPUs can come up or down during this. */
86ef5c9a	483	get_online_cpus();
eeec4fad	484	ret = __stop_machine(fn, data, cpus);
86ef5c9a	485	put_online_cpus();
1da177e4 LT	486	return ret;
1da177e4 LT	487	}
eeec4fad	488	EXPORT_SYMBOL_GPL(stop_machine);
bbf1bb3e TH	489
bbf1bb3e TH	490	#endif /* CONFIG_STOP_MACHINE */