[linux.git] / mm / truncate.c

/*
 * mm/truncate.c - code for taking down pages from address_spaces
 *
 * Copyright (C) 2002, Linus Torvalds
 *
 * 10Sep2002	[email protected]
 *		Initial version.
 */

#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/buffer_head.h>	/* grr. try_to_release_page,
				   do_invalidatepage */


static inline void truncate_partial_page(struct page *page, unsigned partial)
{
	memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
	if (PagePrivate(page))
		do_invalidatepage(page, partial);
}

/*
 * If truncate cannot remove the fs-private metadata from the page, the page
 * becomes anonymous.  It will be left on the LRU and may even be mapped into
 * user pagetables if we're racing with filemap_nopage().
 *
 * We need to bale out if page->mapping is no longer equal to the original
 * mapping.  This happens a) when the VM reclaimed the page while we waited on
 * its lock, b) when a concurrent invalidate_inode_pages got there first and
 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
 */
static void
truncate_complete_page(struct address_space *mapping, struct page *page)
{
	if (page->mapping != mapping)
		return;

	if (PagePrivate(page))
		do_invalidatepage(page, 0);

	clear_page_dirty(page);
	ClearPageUptodate(page);
	ClearPageMappedToDisk(page);
	remove_from_page_cache(page);
	page_cache_release(page);	/* pagecache ref */
}

/*
 * This is for invalidate_inode_pages().  That function can be called at
 * any time, and is not supposed to throw away dirty pages.  But pages can
 * be marked dirty at any time too.  So we re-check the dirtiness inside
 * ->tree_lock.  That provides exclusion against the __set_page_dirty
 * functions.
 *
 * Returns non-zero if the page was successfully invalidated.
 */
static int
invalidate_complete_page(struct address_space *mapping, struct page *page)
{
	if (page->mapping != mapping)
		return 0;

	if (PagePrivate(page) && !try_to_release_page(page, 0))
		return 0;

	write_lock_irq(&mapping->tree_lock);
	if (PageDirty(page)) {
		write_unlock_irq(&mapping->tree_lock);
		return 0;
	}

	BUG_ON(PagePrivate(page));
	__remove_from_page_cache(page);
	write_unlock_irq(&mapping->tree_lock);
	ClearPageUptodate(page);
	page_cache_release(page);	/* pagecache ref */
	return 1;
}

/**
 * truncate_inode_pages - truncate *all* the pages from an offset
 * @mapping: mapping to truncate
 * @lstart: offset from which to truncate
 *
 * Truncate the page cache at a set offset, removing the pages that are beyond
 * that offset (and zeroing out partial pages).
 *
 * Truncate takes two passes - the first pass is nonblocking.  It will not
 * block on page locks and it will not block on writeback.  The second pass
 * will wait.  This is to prevent as much IO as possible in the affected region.
 * The first pass will remove most pages, so the search cost of the second pass
 * is low.
 *
 * When looking at page->index outside the page lock we need to be careful to
 * copy it into a local to avoid races (it could change at any time).
 *
 * We pass down the cache-hot hint to the page freeing code.  Even if the
 * mapping is large, it is probably the case that the final pages are the most
 * recently touched, and freeing happens in ascending file offset order.
 *
 * Called under (and serialised by) inode->i_sem.
 */
void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
{
	const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
	const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
	struct pagevec pvec;
	pgoff_t next;
	int i;

	if (mapping->nrpages == 0)
		return;

	pagevec_init(&pvec, 0);
	next = start;
	while (pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];
			pgoff_t page_index = page->index;

			if (page_index > next)
				next = page_index;
			next++;
			if (TestSetPageLocked(page))
				continue;
			if (PageWriteback(page)) {
				unlock_page(page);
				continue;
			}
			truncate_complete_page(mapping, page);
			unlock_page(page);
		}
		pagevec_release(&pvec);
		cond_resched();
	}

	if (partial) {
		struct page *page = find_lock_page(mapping, start - 1);
		if (page) {
			wait_on_page_writeback(page);
			truncate_partial_page(page, partial);
			unlock_page(page);
			page_cache_release(page);
		}
	}

	next = start;
	for ( ; ; ) {
		cond_resched();
		if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
			if (next == start)
				break;
			next = start;
			continue;
		}
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

			lock_page(page);
			wait_on_page_writeback(page);
			if (page->index > next)
				next = page->index;
			next++;
			truncate_complete_page(mapping, page);
			unlock_page(page);
		}
		pagevec_release(&pvec);
	}
}

EXPORT_SYMBOL(truncate_inode_pages);

/**
 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
 * @mapping: the address_space which holds the pages to invalidate
 * @start: the offset 'from' which to invalidate
 * @end: the offset 'to' which to invalidate (inclusive)
 *
 * This function only removes the unlocked pages, if you want to
 * remove all the pages of one inode, you must call truncate_inode_pages.
 *
 * invalidate_mapping_pages() will not block on IO activity. It will not
 * invalidate pages which are dirty, locked, under writeback or mapped into
 * pagetables.
 */
unsigned long invalidate_mapping_pages(struct address_space *mapping,
				pgoff_t start, pgoff_t end)
{
	struct pagevec pvec;
	pgoff_t next = start;
	unsigned long ret = 0;
	int i;

	pagevec_init(&pvec, 0);
	while (next <= end &&
			pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

			if (TestSetPageLocked(page)) {
				next++;
				continue;
			}
			if (page->index > next)
				next = page->index;
			next++;
			if (PageDirty(page) || PageWriteback(page))
				goto unlock;
			if (page_mapped(page))
				goto unlock;
			ret += invalidate_complete_page(mapping, page);
unlock:
			unlock_page(page);
			if (next > end)
				break;
		}
		pagevec_release(&pvec);
		cond_resched();
	}
	return ret;
}

unsigned long invalidate_inode_pages(struct address_space *mapping)
{
	return invalidate_mapping_pages(mapping, 0, ~0UL);
}

EXPORT_SYMBOL(invalidate_inode_pages);

/**
 * invalidate_inode_pages2_range - remove range of pages from an address_space
 * @mapping: the address_space
 * @start: the page offset 'from' which to invalidate
 * @end: the page offset 'to' which to invalidate (inclusive)
 *
 * Any pages which are found to be mapped into pagetables are unmapped prior to
 * invalidation.
 *
 * Returns -EIO if any pages could not be invalidated.
 */
int invalidate_inode_pages2_range(struct address_space *mapping,
				  pgoff_t start, pgoff_t end)
{
	struct pagevec pvec;
	pgoff_t next;
	int i;
	int ret = 0;
	int did_range_unmap = 0;
	int wrapped = 0;

	pagevec_init(&pvec, 0);
	next = start;
	while (next <= end && !ret && !wrapped &&
		pagevec_lookup(&pvec, mapping, next,
			min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
		for (i = 0; !ret && i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];
			pgoff_t page_index;
			int was_dirty;

			lock_page(page);
			if (page->mapping != mapping) {
				unlock_page(page);
				continue;
			}
			page_index = page->index;
			next = page_index + 1;
			if (next == 0)
				wrapped = 1;
			if (page_index > end) {
				unlock_page(page);
				break;
			}
			wait_on_page_writeback(page);
			while (page_mapped(page)) {
				if (!did_range_unmap) {
					/*
					 * Zap the rest of the file in one hit.
					 */
					unmap_mapping_range(mapping,
					    page_index << PAGE_CACHE_SHIFT,
					    (end - page_index + 1)
							<< PAGE_CACHE_SHIFT,
					    0);
					did_range_unmap = 1;
				} else {
					/*
					 * Just zap this page
					 */
					unmap_mapping_range(mapping,
					  page_index << PAGE_CACHE_SHIFT,
					  PAGE_CACHE_SIZE, 0);
				}
			}
			was_dirty = test_clear_page_dirty(page);
			if (!invalidate_complete_page(mapping, page)) {
				if (was_dirty)
					set_page_dirty(page);
				ret = -EIO;
			}
			unlock_page(page);
		}
		pagevec_release(&pvec);
		cond_resched();
	}
	return ret;
}
EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);

/**
 * invalidate_inode_pages2 - remove all pages from an address_space
 * @mapping: the address_space
 *
 * Any pages which are found to be mapped into pagetables are unmapped prior to
 * invalidation.
 *
 * Returns -EIO if any pages could not be invalidated.
 */
int invalidate_inode_pages2(struct address_space *mapping)
{
	return invalidate_inode_pages2_range(mapping, 0, -1);
}
EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* mm/truncate.c - code for taking down pages from address_spaces
	3	*
	4	* Copyright (C) 2002, Linus Torvalds
	5	*
	6	* 10Sep2002 [email protected]
	7	* Initial version.
	8	*/
	9
	10	#include <linux/kernel.h>
	11	#include <linux/mm.h>
	12	#include <linux/module.h>
	13	#include <linux/pagemap.h>
	14	#include <linux/pagevec.h>
	15	#include <linux/buffer_head.h> /* grr. try_to_release_page,
aaa4059b	16	do_invalidatepage */
1da177e4 LT	17
1da177e4 LT	18
1da177e4 LT	19	static inline void truncate_partial_page(struct page *page, unsigned partial)
	20	{
	21	memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
	22	if (PagePrivate(page))
	23	do_invalidatepage(page, partial);
	24	}
	25
	26	/*
	27	* If truncate cannot remove the fs-private metadata from the page, the page
	28	* becomes anonymous. It will be left on the LRU and may even be mapped into
	29	* user pagetables if we're racing with filemap_nopage().
	30	*
	31	* We need to bale out if page->mapping is no longer equal to the original
	32	* mapping. This happens a) when the VM reclaimed the page while we waited on
	33	* its lock, b) when a concurrent invalidate_inode_pages got there first and
	34	* c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
	35	*/
	36	static void
	37	truncate_complete_page(struct address_space mapping, struct page page)
	38	{
	39	if (page->mapping != mapping)
	40	return;
	41
	42	if (PagePrivate(page))
	43	do_invalidatepage(page, 0);
	44
	45	clear_page_dirty(page);
	46	ClearPageUptodate(page);
	47	ClearPageMappedToDisk(page);
	48	remove_from_page_cache(page);
	49	page_cache_release(page); /* pagecache ref */
	50	}
	51
	52	/*
	53	* This is for invalidate_inode_pages(). That function can be called at
	54	* any time, and is not supposed to throw away dirty pages. But pages can
	55	* be marked dirty at any time too. So we re-check the dirtiness inside
	56	* ->tree_lock. That provides exclusion against the __set_page_dirty
	57	* functions.
	58	*
	59	* Returns non-zero if the page was successfully invalidated.
	60	*/
	61	static int
	62	invalidate_complete_page(struct address_space mapping, struct page page)
	63	{
	64	if (page->mapping != mapping)
	65	return 0;
	66
	67	if (PagePrivate(page) && !try_to_release_page(page, 0))
	68	return 0;
	69
	70	write_lock_irq(&mapping->tree_lock);
	71	if (PageDirty(page)) {
	72	write_unlock_irq(&mapping->tree_lock);
	73	return 0;
	74	}
	75
	76	BUG_ON(PagePrivate(page));
	77	__remove_from_page_cache(page);
	78	write_unlock_irq(&mapping->tree_lock);
	79	ClearPageUptodate(page);
	80	page_cache_release(page); /* pagecache ref */
	81	return 1;
	82	}
83
84	/**
85	* truncate_inode_pages - truncate all the pages from an offset
86	* @mapping: mapping to truncate
87	* @lstart: offset from which to truncate
88	*
89	* Truncate the page cache at a set offset, removing the pages that are beyond
90	* that offset (and zeroing out partial pages).
91	*
92	* Truncate takes two passes - the first pass is nonblocking. It will not
93	* block on page locks and it will not block on writeback. The second pass
94	* will wait. This is to prevent as much IO as possible in the affected region.
95	* The first pass will remove most pages, so the search cost of the second pass
96	* is low.
97	*
98	* When looking at page->index outside the page lock we need to be careful to
99	* copy it into a local to avoid races (it could change at any time).
100	*
101	* We pass down the cache-hot hint to the page freeing code. Even if the
102	* mapping is large, it is probably the case that the final pages are the most
103	* recently touched, and freeing happens in ascending file offset order.
104	*
105	* Called under (and serialised by) inode->i_sem.
106	*/
107	void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
108	{
109	const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
110	const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
111	struct pagevec pvec;
112	pgoff_t next;
113	int i;
114
115	if (mapping->nrpages == 0)
116	return;
117
118	pagevec_init(&pvec, 0);
119	next = start;
120	while (pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
121	for (i = 0; i < pagevec_count(&pvec); i++) {
122	struct page *page = pvec.pages[i];
123	pgoff_t page_index = page->index;
124
125	if (page_index > next)
126	next = page_index;
127	next++;
128	if (TestSetPageLocked(page))
129	continue;
130	if (PageWriteback(page)) {
131	unlock_page(page);
132	continue;
133	}
134	truncate_complete_page(mapping, page);
135	unlock_page(page);
136	}
137	pagevec_release(&pvec);
138	cond_resched();
139	}
140
141	if (partial) {
142	struct page *page = find_lock_page(mapping, start - 1);
143	if (page) {
144	wait_on_page_writeback(page);
145	truncate_partial_page(page, partial);
146	unlock_page(page);
147	page_cache_release(page);
148	}
149	}
150
151	next = start;
152	for ( ; ; ) {
153	cond_resched();
154	if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
155	if (next == start)
156	break;
157	next = start;
158	continue;
159	}
160	for (i = 0; i < pagevec_count(&pvec); i++) {
161	struct page *page = pvec.pages[i];
162
163	lock_page(page);
164	wait_on_page_writeback(page);
165	if (page->index > next)
166	next = page->index;
167	next++;
168	truncate_complete_page(mapping, page);
169	unlock_page(page);
170	}
171	pagevec_release(&pvec);
172	}
173	}
174
175	EXPORT_SYMBOL(truncate_inode_pages);
176
177	/**
178	* invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
179	* @mapping: the address_space which holds the pages to invalidate
180	* @start: the offset 'from' which to invalidate
181	* @end: the offset 'to' which to invalidate (inclusive)
182	*
183	* This function only removes the unlocked pages, if you want to
184	* remove all the pages of one inode, you must call truncate_inode_pages.
185	*
186	* invalidate_mapping_pages() will not block on IO activity. It will not
187	* invalidate pages which are dirty, locked, under writeback or mapped into
188	* pagetables.
189	*/
190	unsigned long invalidate_mapping_pages(struct address_space *mapping,
191	pgoff_t start, pgoff_t end)
192	{
193	struct pagevec pvec;
194	pgoff_t next = start;
195	unsigned long ret = 0;
196	int i;
197
198	pagevec_init(&pvec, 0);
199	while (next <= end &&
200	pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
201	for (i = 0; i < pagevec_count(&pvec); i++) {
202	struct page *page = pvec.pages[i];
203
204	if (TestSetPageLocked(page)) {
205	next++;
206	continue;
207	}
208	if (page->index > next)
209	next = page->index;
210	next++;
211	if (PageDirty(page) \|\| PageWriteback(page))
212	goto unlock;
213	if (page_mapped(page))
214	goto unlock;
215	ret += invalidate_complete_page(mapping, page);
216	unlock:
217	unlock_page(page);
218	if (next > end)
219	break;
220	}
221	pagevec_release(&pvec);
222	cond_resched();
223	}
224	return ret;
225	}
226
227	unsigned long invalidate_inode_pages(struct address_space *mapping)
228	{
229	return invalidate_mapping_pages(mapping, 0, ~0UL);
230	}
231
232	EXPORT_SYMBOL(invalidate_inode_pages);
233
234	/**
235	* invalidate_inode_pages2_range - remove range of pages from an address_space
67be2dd1	236	* @mapping: the address_space
1da177e4 LT	237	* @start: the page offset 'from' which to invalidate
	238	* @end: the page offset 'to' which to invalidate (inclusive)
	239	*
	240	* Any pages which are found to be mapped into pagetables are unmapped prior to
	241	* invalidation.
	242	*
	243	* Returns -EIO if any pages could not be invalidated.
	244	*/
	245	int invalidate_inode_pages2_range(struct address_space *mapping,
	246	pgoff_t start, pgoff_t end)
	247	{
	248	struct pagevec pvec;
	249	pgoff_t next;
	250	int i;
	251	int ret = 0;
	252	int did_range_unmap = 0;
	253	int wrapped = 0;
	254
	255	pagevec_init(&pvec, 0);
	256	next = start;
	257	while (next <= end && !ret && !wrapped &&
	258	pagevec_lookup(&pvec, mapping, next,
	259	min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
	260	for (i = 0; !ret && i < pagevec_count(&pvec); i++) {
	261	struct page *page = pvec.pages[i];
	262	pgoff_t page_index;
	263	int was_dirty;
	264
	265	lock_page(page);
	266	if (page->mapping != mapping) {
	267	unlock_page(page);
	268	continue;
	269	}
	270	page_index = page->index;
	271	next = page_index + 1;
	272	if (next == 0)
	273	wrapped = 1;
	274	if (page_index > end) {
	275	unlock_page(page);
	276	break;
	277	}
	278	wait_on_page_writeback(page);
	279	while (page_mapped(page)) {
	280	if (!did_range_unmap) {
	281	/*
	282	* Zap the rest of the file in one hit.
	283	*/
	284	unmap_mapping_range(mapping,
	285	page_index << PAGE_CACHE_SHIFT,
	286	(end - page_index + 1)
	287	<< PAGE_CACHE_SHIFT,
	288	0);
	289	did_range_unmap = 1;
	290	} else {
	291	/*
	292	* Just zap this page
	293	*/
	294	unmap_mapping_range(mapping,
	295	page_index << PAGE_CACHE_SHIFT,
	296	PAGE_CACHE_SIZE, 0);
	297	}
	298	}
	299	was_dirty = test_clear_page_dirty(page);
	300	if (!invalidate_complete_page(mapping, page)) {
301	if (was_dirty)
302	set_page_dirty(page);
303	ret = -EIO;
304	}
305	unlock_page(page);
306	}
307	pagevec_release(&pvec);
308	cond_resched();
309	}
310	return ret;
311	}
312	EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
313
314	/**
315	* invalidate_inode_pages2 - remove all pages from an address_space
67be2dd1	316	* @mapping: the address_space
1da177e4 LT	317	*
	318	* Any pages which are found to be mapped into pagetables are unmapped prior to
	319	* invalidation.
	320	*
	321	* Returns -EIO if any pages could not be invalidated.
	322	*/
	323	int invalidate_inode_pages2(struct address_space *mapping)
	324	{
	325	return invalidate_inode_pages2_range(mapping, 0, -1);
	326	}
	327	EXPORT_SYMBOL_GPL(invalidate_inode_pages2);