[linux.git] / mm / pdflush.c

/*
 * mm/pdflush.c - worker threads for writing back filesystem data
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * 09Apr2002	[email protected]
 *		Initial version
 * 29Feb2004	[email protected]
 *		Move worker thread creation to kthread to avoid chewing
 *		up stack space with nested calls to kernel_thread.
 */

#include <linux/sched.h>
#include <linux/list.h>
#include <linux/signal.h>
#include <linux/spinlock.h>
#include <linux/gfp.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/fs.h>		// Needed by writeback.h
#include <linux/writeback.h>	// Prototypes pdflush_operation()
#include <linux/kthread.h>
#include <linux/cpuset.h>
#include <linux/freezer.h>


/*
 * Minimum and maximum number of pdflush instances
 */
#define MIN_PDFLUSH_THREADS	2
#define MAX_PDFLUSH_THREADS	8

static void start_one_pdflush_thread(void);


/*
 * The pdflush threads are worker threads for writing back dirty data.
 * Ideally, we'd like one thread per active disk spindle.  But the disk
 * topology is very hard to divine at this level.   Instead, we take
 * care in various places to prevent more than one pdflush thread from
 * performing writeback against a single filesystem.  pdflush threads
 * have the PF_FLUSHER flag set in current->flags to aid in this.
 */

/*
 * All the pdflush threads.  Protected by pdflush_lock
 */
static LIST_HEAD(pdflush_list);
static DEFINE_SPINLOCK(pdflush_lock);

/*
 * The count of currently-running pdflush threads.  Protected
 * by pdflush_lock.
 *
 * Readable by sysctl, but not writable.  Published to userspace at
 * /proc/sys/vm/nr_pdflush_threads.
 */
int nr_pdflush_threads = 0;

/*
 * The time at which the pdflush thread pool last went empty
 */
static unsigned long last_empty_jifs;

/*
 * The pdflush thread.
 *
 * Thread pool management algorithm:
 * 
 * - The minimum and maximum number of pdflush instances are bound
 *   by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
 * 
 * - If there have been no idle pdflush instances for 1 second, create
 *   a new one.
 * 
 * - If the least-recently-went-to-sleep pdflush thread has been asleep
 *   for more than one second, terminate a thread.
 */

/*
 * A structure for passing work to a pdflush thread.  Also for passing
 * state information between pdflush threads.  Protected by pdflush_lock.
 */
struct pdflush_work {
	struct task_struct *who;	/* The thread */
	void (*fn)(unsigned long);	/* A callback function */
	unsigned long arg0;		/* An argument to the callback */
	struct list_head list;		/* On pdflush_list, when idle */
	unsigned long when_i_went_to_sleep;
};

static int __pdflush(struct pdflush_work *my_work)
{
	current->flags |= PF_FLUSHER | PF_SWAPWRITE;
	set_freezable();
	my_work->fn = NULL;
	my_work->who = current;
	INIT_LIST_HEAD(&my_work->list);

	spin_lock_irq(&pdflush_lock);
	nr_pdflush_threads++;
	for ( ; ; ) {
		struct pdflush_work *pdf;

		set_current_state(TASK_INTERRUPTIBLE);
		list_move(&my_work->list, &pdflush_list);
		my_work->when_i_went_to_sleep = jiffies;
		spin_unlock_irq(&pdflush_lock);
		schedule();
		try_to_freeze();
		spin_lock_irq(&pdflush_lock);
		if (!list_empty(&my_work->list)) {
			/*
			 * Someone woke us up, but without removing our control
			 * structure from the global list.  swsusp will do this
			 * in try_to_freeze()->refrigerator().  Handle it.
			 */
			my_work->fn = NULL;
			continue;
		}
		if (my_work->fn == NULL) {
			printk("pdflush: bogus wakeup\n");
			continue;
		}
		spin_unlock_irq(&pdflush_lock);

		(*my_work->fn)(my_work->arg0);

		/*
		 * Thread creation: For how long have there been zero
		 * available threads?
		 */
		if (jiffies - last_empty_jifs > 1 * HZ) {
			/* unlocked list_empty() test is OK here */
			if (list_empty(&pdflush_list)) {
				/* unlocked test is OK here */
				if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
					start_one_pdflush_thread();
			}
		}

		spin_lock_irq(&pdflush_lock);
		my_work->fn = NULL;

		/*
		 * Thread destruction: For how long has the sleepiest
		 * thread slept?
		 */
		if (list_empty(&pdflush_list))
			continue;
		if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
			continue;
		pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
		if (jiffies - pdf->when_i_went_to_sleep > 1 * HZ) {
			/* Limit exit rate */
			pdf->when_i_went_to_sleep = jiffies;
			break;					/* exeunt */
		}
	}
	nr_pdflush_threads--;
	spin_unlock_irq(&pdflush_lock);
	return 0;
}

/*
 * Of course, my_work wants to be just a local in __pdflush().  It is
 * separated out in this manner to hopefully prevent the compiler from
 * performing unfortunate optimisations against the auto variables.  Because
 * these are visible to other tasks and CPUs.  (No problem has actually
 * been observed.  This is just paranoia).
 */
static int pdflush(void *dummy)
{
	struct pdflush_work my_work;
	cpumask_t cpus_allowed;

	/*
	 * pdflush can spend a lot of time doing encryption via dm-crypt.  We
	 * don't want to do that at keventd's priority.
	 */
	set_user_nice(current, 0);

	/*
	 * Some configs put our parent kthread in a limited cpuset,
	 * which kthread() overrides, forcing cpus_allowed == CPU_MASK_ALL.
	 * Our needs are more modest - cut back to our cpusets cpus_allowed.
	 * This is needed as pdflush's are dynamically created and destroyed.
	 * The boottime pdflush's are easily placed w/o these 2 lines.
	 */
	cpus_allowed = cpuset_cpus_allowed(current);
	set_cpus_allowed(current, cpus_allowed);

	return __pdflush(&my_work);
}

/*
 * Attempt to wake up a pdflush thread, and get it to do some work for you.
 * Returns zero if it indeed managed to find a worker thread, and passed your
 * payload to it.
 */
int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
{
	unsigned long flags;
	int ret = 0;

	BUG_ON(fn == NULL);	/* Hard to diagnose if it's deferred */

	spin_lock_irqsave(&pdflush_lock, flags);
	if (list_empty(&pdflush_list)) {
		spin_unlock_irqrestore(&pdflush_lock, flags);
		ret = -1;
	} else {
		struct pdflush_work *pdf;

		pdf = list_entry(pdflush_list.next, struct pdflush_work, list);
		list_del_init(&pdf->list);
		if (list_empty(&pdflush_list))
			last_empty_jifs = jiffies;
		pdf->fn = fn;
		pdf->arg0 = arg0;
		wake_up_process(pdf->who);
		spin_unlock_irqrestore(&pdflush_lock, flags);
	}
	return ret;
}

static void start_one_pdflush_thread(void)
{
	kthread_run(pdflush, NULL, "pdflush");
}

static int __init pdflush_init(void)
{
	int i;

	for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
		start_one_pdflush_thread();
	return 0;
}

module_init(pdflush_init);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* mm/pdflush.c - worker threads for writing back filesystem data
	3	*
	4	* Copyright (C) 2002, Linus Torvalds.
	5	*
	6	* 09Apr2002 [email protected]
	7	* Initial version
	8	* 29Feb2004 [email protected]
	9	* Move worker thread creation to kthread to avoid chewing
	10	* up stack space with nested calls to kernel_thread.
	11	*/
	12
	13	#include <linux/sched.h>
	14	#include <linux/list.h>
	15	#include <linux/signal.h>
	16	#include <linux/spinlock.h>
	17	#include <linux/gfp.h>
	18	#include <linux/init.h>
	19	#include <linux/module.h>
	20	#include <linux/fs.h> // Needed by writeback.h
	21	#include <linux/writeback.h> // Prototypes pdflush_operation()
	22	#include <linux/kthread.h>
28a42b9e	23	#include <linux/cpuset.h>
7dfb7103	24	#include <linux/freezer.h>
1da177e4 LT	25
	26
	27	/*
	28	* Minimum and maximum number of pdflush instances
	29	*/
	30	#define MIN_PDFLUSH_THREADS 2
	31	#define MAX_PDFLUSH_THREADS 8
	32
	33	static void start_one_pdflush_thread(void);
	34
	35
	36	/*
	37	* The pdflush threads are worker threads for writing back dirty data.
	38	* Ideally, we'd like one thread per active disk spindle. But the disk
	39	* topology is very hard to divine at this level. Instead, we take
	40	* care in various places to prevent more than one pdflush thread from
	41	* performing writeback against a single filesystem. pdflush threads
	42	* have the PF_FLUSHER flag set in current->flags to aid in this.
	43	*/
	44
	45	/*
	46	* All the pdflush threads. Protected by pdflush_lock
	47	*/
	48	static LIST_HEAD(pdflush_list);
	49	static DEFINE_SPINLOCK(pdflush_lock);
	50
	51	/*
	52	* The count of currently-running pdflush threads. Protected
	53	* by pdflush_lock.
	54	*
	55	* Readable by sysctl, but not writable. Published to userspace at
	56	* /proc/sys/vm/nr_pdflush_threads.
	57	*/
	58	int nr_pdflush_threads = 0;
	59
	60	/*
	61	* The time at which the pdflush thread pool last went empty
	62	*/
	63	static unsigned long last_empty_jifs;
	64
	65	/*
	66	* The pdflush thread.
	67	*
	68	* Thread pool management algorithm:
	69	*
	70	* - The minimum and maximum number of pdflush instances are bound
	71	* by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
	72	*
	73	* - If there have been no idle pdflush instances for 1 second, create
	74	* a new one.
	75	*
	76	* - If the least-recently-went-to-sleep pdflush thread has been asleep
	77	* for more than one second, terminate a thread.
	78	*/
	79
	80	/*
	81	* A structure for passing work to a pdflush thread. Also for passing
	82	* state information between pdflush threads. Protected by pdflush_lock.
	83	*/
	84	struct pdflush_work {
	85	struct task_struct who; / The thread */
	86	void (fn)(unsigned long); / A callback function */
	87	unsigned long arg0; /* An argument to the callback */
	88	struct list_head list; /* On pdflush_list, when idle */
89	unsigned long when_i_went_to_sleep;
90	};
91
92	static int __pdflush(struct pdflush_work *my_work)
93	{
930d9152	94	current->flags \|= PF_FLUSHER \| PF_SWAPWRITE;
83144186	95	set_freezable();
1da177e4 LT	96	my_work->fn = NULL;
	97	my_work->who = current;
	98	INIT_LIST_HEAD(&my_work->list);
	99
	100	spin_lock_irq(&pdflush_lock);
	101	nr_pdflush_threads++;
	102	for ( ; ; ) {
	103	struct pdflush_work *pdf;
	104
	105	set_current_state(TASK_INTERRUPTIBLE);
	106	list_move(&my_work->list, &pdflush_list);
	107	my_work->when_i_went_to_sleep = jiffies;
	108	spin_unlock_irq(&pdflush_lock);
1da177e4	109	schedule();
d616e09a	110	try_to_freeze();
1da177e4 LT	111	spin_lock_irq(&pdflush_lock);
1da177e4 LT	112	if (!list_empty(&my_work->list)) {
d616e09a AM	113	/*
	114	* Someone woke us up, but without removing our control
	115	* structure from the global list. swsusp will do this
	116	* in try_to_freeze()->refrigerator(). Handle it.
	117	*/
1da177e4 LT	118	my_work->fn = NULL;
	119	continue;
	120	}
	121	if (my_work->fn == NULL) {
d616e09a	122	printk("pdflush: bogus wakeup\n");
1da177e4 LT	123	continue;
	124	}
	125	spin_unlock_irq(&pdflush_lock);
	126
	127	(*my_work->fn)(my_work->arg0);
	128
	129	/*
	130	* Thread creation: For how long have there been zero
	131	* available threads?
	132	*/
	133	if (jiffies - last_empty_jifs > 1 * HZ) {
	134	/* unlocked list_empty() test is OK here */
	135	if (list_empty(&pdflush_list)) {
	136	/* unlocked test is OK here */
	137	if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
	138	start_one_pdflush_thread();
	139	}
	140	}
	141
	142	spin_lock_irq(&pdflush_lock);
	143	my_work->fn = NULL;
	144
	145	/*
	146	* Thread destruction: For how long has the sleepiest
	147	* thread slept?
	148	*/
	149	if (list_empty(&pdflush_list))
	150	continue;
	151	if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
	152	continue;
	153	pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
	154	if (jiffies - pdf->when_i_went_to_sleep > 1 * HZ) {
	155	/* Limit exit rate */
	156	pdf->when_i_went_to_sleep = jiffies;
	157	break; /* exeunt */
	158	}
	159	}
	160	nr_pdflush_threads--;
	161	spin_unlock_irq(&pdflush_lock);
	162	return 0;
	163	}
	164
	165	/*
	166	* Of course, my_work wants to be just a local in __pdflush(). It is
	167	* separated out in this manner to hopefully prevent the compiler from
	168	* performing unfortunate optimisations against the auto variables. Because
	169	* these are visible to other tasks and CPUs. (No problem has actually
	170	* been observed. This is just paranoia).
	171	*/
	172	static int pdflush(void *dummy)
	173	{
	174	struct pdflush_work my_work;
28a42b9e	175	cpumask_t cpus_allowed;
1da177e4 LT	176
	177	/*
	178	* pdflush can spend a lot of time doing encryption via dm-crypt. We
	179	* don't want to do that at keventd's priority.
	180	*/
	181	set_user_nice(current, 0);
28a42b9e PJ	182
	183	/*
	184	* Some configs put our parent kthread in a limited cpuset,
	185	* which kthread() overrides, forcing cpus_allowed == CPU_MASK_ALL.
	186	* Our needs are more modest - cut back to our cpusets cpus_allowed.
	187	* This is needed as pdflush's are dynamically created and destroyed.
	188	* The boottime pdflush's are easily placed w/o these 2 lines.
	189	*/
	190	cpus_allowed = cpuset_cpus_allowed(current);
	191	set_cpus_allowed(current, cpus_allowed);
	192
1da177e4 LT	193	return __pdflush(&my_work);
	194	}
	195
	196	/*
	197	* Attempt to wake up a pdflush thread, and get it to do some work for you.
	198	* Returns zero if it indeed managed to find a worker thread, and passed your
	199	* payload to it.
	200	*/
	201	int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
	202	{
	203	unsigned long flags;
	204	int ret = 0;
	205
125e1874	206	BUG_ON(fn == NULL); /* Hard to diagnose if it's deferred */
1da177e4 LT	207
	208	spin_lock_irqsave(&pdflush_lock, flags);
	209	if (list_empty(&pdflush_list)) {
	210	spin_unlock_irqrestore(&pdflush_lock, flags);
	211	ret = -1;
	212	} else {
	213	struct pdflush_work *pdf;
	214
	215	pdf = list_entry(pdflush_list.next, struct pdflush_work, list);
	216	list_del_init(&pdf->list);
	217	if (list_empty(&pdflush_list))
	218	last_empty_jifs = jiffies;
	219	pdf->fn = fn;
	220	pdf->arg0 = arg0;
	221	wake_up_process(pdf->who);
	222	spin_unlock_irqrestore(&pdflush_lock, flags);
	223	}
	224	return ret;
	225	}
	226
	227	static void start_one_pdflush_thread(void)
	228	{
	229	kthread_run(pdflush, NULL, "pdflush");
	230	}
	231
	232	static int __init pdflush_init(void)
	233	{
	234	int i;
	235
	236	for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
	237	start_one_pdflush_thread();
	238	return 0;
	239	}
	240
	241	module_init(pdflush_init);