[linux.git] / mm / pdflush.c

/*
 * mm/pdflush.c - worker threads for writing back filesystem data
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * 09Apr2002	[email protected]
 *		Initial version
 * 29Feb2004	[email protected]
 *		Move worker thread creation to kthread to avoid chewing
 *		up stack space with nested calls to kernel_thread.
 */

#include <linux/sched.h>
#include <linux/list.h>
#include <linux/signal.h>
#include <linux/spinlock.h>
#include <linux/gfp.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/fs.h>		// Needed by writeback.h
#include <linux/writeback.h>	// Prototypes pdflush_operation()
#include <linux/kthread.h>
#include <linux/cpuset.h>


/*
 * Minimum and maximum number of pdflush instances
 */
#define MIN_PDFLUSH_THREADS	2
#define MAX_PDFLUSH_THREADS	8

static void start_one_pdflush_thread(void);


/*
 * The pdflush threads are worker threads for writing back dirty data.
 * Ideally, we'd like one thread per active disk spindle.  But the disk
 * topology is very hard to divine at this level.   Instead, we take
 * care in various places to prevent more than one pdflush thread from
 * performing writeback against a single filesystem.  pdflush threads
 * have the PF_FLUSHER flag set in current->flags to aid in this.
 */

/*
 * All the pdflush threads.  Protected by pdflush_lock
 */
static LIST_HEAD(pdflush_list);
static DEFINE_SPINLOCK(pdflush_lock);

/*
 * The count of currently-running pdflush threads.  Protected
 * by pdflush_lock.
 *
 * Readable by sysctl, but not writable.  Published to userspace at
 * /proc/sys/vm/nr_pdflush_threads.
 */
int nr_pdflush_threads = 0;

/*
 * The time at which the pdflush thread pool last went empty
 */
static unsigned long last_empty_jifs;

/*
 * The pdflush thread.
 *
 * Thread pool management algorithm:
 * 
 * - The minimum and maximum number of pdflush instances are bound
 *   by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
 * 
 * - If there have been no idle pdflush instances for 1 second, create
 *   a new one.
 * 
 * - If the least-recently-went-to-sleep pdflush thread has been asleep
 *   for more than one second, terminate a thread.
 */

/*
 * A structure for passing work to a pdflush thread.  Also for passing
 * state information between pdflush threads.  Protected by pdflush_lock.
 */
struct pdflush_work {
	struct task_struct *who;	/* The thread */
	void (*fn)(unsigned long);	/* A callback function */
	unsigned long arg0;		/* An argument to the callback */
	struct list_head list;		/* On pdflush_list, when idle */
	unsigned long when_i_went_to_sleep;
};

static int __pdflush(struct pdflush_work *my_work)
{
	current->flags |= PF_FLUSHER | PF_SWAPWRITE;
	my_work->fn = NULL;
	my_work->who = current;
	INIT_LIST_HEAD(&my_work->list);

	spin_lock_irq(&pdflush_lock);
	nr_pdflush_threads++;
	for ( ; ; ) {
		struct pdflush_work *pdf;

		set_current_state(TASK_INTERRUPTIBLE);
		list_move(&my_work->list, &pdflush_list);
		my_work->when_i_went_to_sleep = jiffies;
		spin_unlock_irq(&pdflush_lock);
		schedule();
		try_to_freeze();
		spin_lock_irq(&pdflush_lock);
		if (!list_empty(&my_work->list)) {
			/*
			 * Someone woke us up, but without removing our control
			 * structure from the global list.  swsusp will do this
			 * in try_to_freeze()->refrigerator().  Handle it.
			 */
			my_work->fn = NULL;
			continue;
		}
		if (my_work->fn == NULL) {
			printk("pdflush: bogus wakeup\n");
			continue;
		}
		spin_unlock_irq(&pdflush_lock);

		(*my_work->fn)(my_work->arg0);

		/*
		 * Thread creation: For how long have there been zero
		 * available threads?
		 */
		if (jiffies - last_empty_jifs > 1 * HZ) {
			/* unlocked list_empty() test is OK here */
			if (list_empty(&pdflush_list)) {
				/* unlocked test is OK here */
				if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
					start_one_pdflush_thread();
			}
		}

		spin_lock_irq(&pdflush_lock);
		my_work->fn = NULL;

		/*
		 * Thread destruction: For how long has the sleepiest
		 * thread slept?
		 */
		if (list_empty(&pdflush_list))
			continue;
		if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
			continue;
		pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
		if (jiffies - pdf->when_i_went_to_sleep > 1 * HZ) {
			/* Limit exit rate */
			pdf->when_i_went_to_sleep = jiffies;
			break;					/* exeunt */
		}
	}
	nr_pdflush_threads--;
	spin_unlock_irq(&pdflush_lock);
	return 0;
}

/*
 * Of course, my_work wants to be just a local in __pdflush().  It is
 * separated out in this manner to hopefully prevent the compiler from
 * performing unfortunate optimisations against the auto variables.  Because
 * these are visible to other tasks and CPUs.  (No problem has actually
 * been observed.  This is just paranoia).
 */
static int pdflush(void *dummy)
{
	struct pdflush_work my_work;
	cpumask_t cpus_allowed;

	/*
	 * pdflush can spend a lot of time doing encryption via dm-crypt.  We
	 * don't want to do that at keventd's priority.
	 */
	set_user_nice(current, 0);

	/*
	 * Some configs put our parent kthread in a limited cpuset,
	 * which kthread() overrides, forcing cpus_allowed == CPU_MASK_ALL.
	 * Our needs are more modest - cut back to our cpusets cpus_allowed.
	 * This is needed as pdflush's are dynamically created and destroyed.
	 * The boottime pdflush's are easily placed w/o these 2 lines.
	 */
	cpus_allowed = cpuset_cpus_allowed(current);
	set_cpus_allowed(current, cpus_allowed);

	return __pdflush(&my_work);
}

/*
 * Attempt to wake up a pdflush thread, and get it to do some work for you.
 * Returns zero if it indeed managed to find a worker thread, and passed your
 * payload to it.
 */
int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
{
	unsigned long flags;
	int ret = 0;

	BUG_ON(fn == NULL);	/* Hard to diagnose if it's deferred */

	spin_lock_irqsave(&pdflush_lock, flags);
	if (list_empty(&pdflush_list)) {
		spin_unlock_irqrestore(&pdflush_lock, flags);
		ret = -1;
	} else {
		struct pdflush_work *pdf;

		pdf = list_entry(pdflush_list.next, struct pdflush_work, list);
		list_del_init(&pdf->list);
		if (list_empty(&pdflush_list))
			last_empty_jifs = jiffies;
		pdf->fn = fn;
		pdf->arg0 = arg0;
		wake_up_process(pdf->who);
		spin_unlock_irqrestore(&pdflush_lock, flags);
	}
	return ret;
}

static void start_one_pdflush_thread(void)
{
	kthread_run(pdflush, NULL, "pdflush");
}

static int __init pdflush_init(void)
{
	int i;

	for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
		start_one_pdflush_thread();
	return 0;
}

module_init(pdflush_init);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* mm/pdflush.c - worker threads for writing back filesystem data
	3	*
	4	* Copyright (C) 2002, Linus Torvalds.
	5	*
	6	* 09Apr2002 [email protected]
	7	* Initial version
	8	* 29Feb2004 [email protected]
	9	* Move worker thread creation to kthread to avoid chewing
	10	* up stack space with nested calls to kernel_thread.
	11	*/
	12
	13	#include <linux/sched.h>
	14	#include <linux/list.h>
	15	#include <linux/signal.h>
	16	#include <linux/spinlock.h>
	17	#include <linux/gfp.h>
	18	#include <linux/init.h>
	19	#include <linux/module.h>
	20	#include <linux/fs.h> // Needed by writeback.h
	21	#include <linux/writeback.h> // Prototypes pdflush_operation()
	22	#include <linux/kthread.h>
28a42b9e	23	#include <linux/cpuset.h>
1da177e4 LT	24
	25
	26	/*
	27	* Minimum and maximum number of pdflush instances
	28	*/
	29	#define MIN_PDFLUSH_THREADS 2
	30	#define MAX_PDFLUSH_THREADS 8
	31
	32	static void start_one_pdflush_thread(void);
	33
	34
	35	/*
	36	* The pdflush threads are worker threads for writing back dirty data.
	37	* Ideally, we'd like one thread per active disk spindle. But the disk
	38	* topology is very hard to divine at this level. Instead, we take
	39	* care in various places to prevent more than one pdflush thread from
	40	* performing writeback against a single filesystem. pdflush threads
	41	* have the PF_FLUSHER flag set in current->flags to aid in this.
	42	*/
	43
	44	/*
	45	* All the pdflush threads. Protected by pdflush_lock
	46	*/
	47	static LIST_HEAD(pdflush_list);
	48	static DEFINE_SPINLOCK(pdflush_lock);
	49
	50	/*
	51	* The count of currently-running pdflush threads. Protected
	52	* by pdflush_lock.
	53	*
	54	* Readable by sysctl, but not writable. Published to userspace at
	55	* /proc/sys/vm/nr_pdflush_threads.
	56	*/
	57	int nr_pdflush_threads = 0;
	58
	59	/*
	60	* The time at which the pdflush thread pool last went empty
	61	*/
	62	static unsigned long last_empty_jifs;
	63
	64	/*
	65	* The pdflush thread.
	66	*
	67	* Thread pool management algorithm:
	68	*
	69	* - The minimum and maximum number of pdflush instances are bound
	70	* by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
	71	*
	72	* - If there have been no idle pdflush instances for 1 second, create
	73	* a new one.
	74	*
	75	* - If the least-recently-went-to-sleep pdflush thread has been asleep
	76	* for more than one second, terminate a thread.
	77	*/
	78
	79	/*
	80	* A structure for passing work to a pdflush thread. Also for passing
	81	* state information between pdflush threads. Protected by pdflush_lock.
	82	*/
	83	struct pdflush_work {
	84	struct task_struct who; / The thread */
	85	void (fn)(unsigned long); / A callback function */
	86	unsigned long arg0; /* An argument to the callback */
	87	struct list_head list; /* On pdflush_list, when idle */
88	unsigned long when_i_went_to_sleep;
89	};
90
91	static int __pdflush(struct pdflush_work *my_work)
92	{
930d9152	93	current->flags \|= PF_FLUSHER \| PF_SWAPWRITE;
1da177e4 LT	94	my_work->fn = NULL;
	95	my_work->who = current;
	96	INIT_LIST_HEAD(&my_work->list);
	97
	98	spin_lock_irq(&pdflush_lock);
	99	nr_pdflush_threads++;
	100	for ( ; ; ) {
	101	struct pdflush_work *pdf;
	102
	103	set_current_state(TASK_INTERRUPTIBLE);
	104	list_move(&my_work->list, &pdflush_list);
	105	my_work->when_i_went_to_sleep = jiffies;
	106	spin_unlock_irq(&pdflush_lock);
1da177e4	107	schedule();
d616e09a	108	try_to_freeze();
1da177e4 LT	109	spin_lock_irq(&pdflush_lock);
1da177e4 LT	110	if (!list_empty(&my_work->list)) {
d616e09a AM	111	/*
	112	* Someone woke us up, but without removing our control
	113	* structure from the global list. swsusp will do this
	114	* in try_to_freeze()->refrigerator(). Handle it.
	115	*/
1da177e4 LT	116	my_work->fn = NULL;
	117	continue;
	118	}
	119	if (my_work->fn == NULL) {
d616e09a	120	printk("pdflush: bogus wakeup\n");
1da177e4 LT	121	continue;
	122	}
	123	spin_unlock_irq(&pdflush_lock);
	124
	125	(*my_work->fn)(my_work->arg0);
	126
	127	/*
	128	* Thread creation: For how long have there been zero
	129	* available threads?
	130	*/
	131	if (jiffies - last_empty_jifs > 1 * HZ) {
	132	/* unlocked list_empty() test is OK here */
	133	if (list_empty(&pdflush_list)) {
	134	/* unlocked test is OK here */
	135	if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
	136	start_one_pdflush_thread();
	137	}
	138	}
	139
	140	spin_lock_irq(&pdflush_lock);
	141	my_work->fn = NULL;
	142
	143	/*
	144	* Thread destruction: For how long has the sleepiest
	145	* thread slept?
	146	*/
	147	if (list_empty(&pdflush_list))
	148	continue;
	149	if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
	150	continue;
	151	pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
	152	if (jiffies - pdf->when_i_went_to_sleep > 1 * HZ) {
	153	/* Limit exit rate */
	154	pdf->when_i_went_to_sleep = jiffies;
	155	break; /* exeunt */
	156	}
	157	}
	158	nr_pdflush_threads--;
	159	spin_unlock_irq(&pdflush_lock);
	160	return 0;
	161	}
	162
	163	/*
	164	* Of course, my_work wants to be just a local in __pdflush(). It is
	165	* separated out in this manner to hopefully prevent the compiler from
	166	* performing unfortunate optimisations against the auto variables. Because
	167	* these are visible to other tasks and CPUs. (No problem has actually
	168	* been observed. This is just paranoia).
	169	*/
	170	static int pdflush(void *dummy)
	171	{
	172	struct pdflush_work my_work;
28a42b9e	173	cpumask_t cpus_allowed;
1da177e4 LT	174
	175	/*
	176	* pdflush can spend a lot of time doing encryption via dm-crypt. We
	177	* don't want to do that at keventd's priority.
	178	*/
	179	set_user_nice(current, 0);
28a42b9e PJ	180
	181	/*
	182	* Some configs put our parent kthread in a limited cpuset,
	183	* which kthread() overrides, forcing cpus_allowed == CPU_MASK_ALL.
	184	* Our needs are more modest - cut back to our cpusets cpus_allowed.
	185	* This is needed as pdflush's are dynamically created and destroyed.
	186	* The boottime pdflush's are easily placed w/o these 2 lines.
	187	*/
	188	cpus_allowed = cpuset_cpus_allowed(current);
	189	set_cpus_allowed(current, cpus_allowed);
	190
1da177e4 LT	191	return __pdflush(&my_work);
	192	}
	193
	194	/*
	195	* Attempt to wake up a pdflush thread, and get it to do some work for you.
	196	* Returns zero if it indeed managed to find a worker thread, and passed your
	197	* payload to it.
	198	*/
	199	int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
	200	{
	201	unsigned long flags;
	202	int ret = 0;
	203
125e1874	204	BUG_ON(fn == NULL); /* Hard to diagnose if it's deferred */
1da177e4 LT	205
	206	spin_lock_irqsave(&pdflush_lock, flags);
	207	if (list_empty(&pdflush_list)) {
	208	spin_unlock_irqrestore(&pdflush_lock, flags);
	209	ret = -1;
	210	} else {
	211	struct pdflush_work *pdf;
	212
	213	pdf = list_entry(pdflush_list.next, struct pdflush_work, list);
	214	list_del_init(&pdf->list);
	215	if (list_empty(&pdflush_list))
	216	last_empty_jifs = jiffies;
	217	pdf->fn = fn;
	218	pdf->arg0 = arg0;
	219	wake_up_process(pdf->who);
	220	spin_unlock_irqrestore(&pdflush_lock, flags);
	221	}
	222	return ret;
	223	}
	224
	225	static void start_one_pdflush_thread(void)
	226	{
	227	kthread_run(pdflush, NULL, "pdflush");
	228	}
	229
	230	static int __init pdflush_init(void)
	231	{
	232	int i;
	233
	234	for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
	235	start_one_pdflush_thread();
	236	return 0;
	237	}
	238
	239	module_init(pdflush_init);