mm: only set page->pfmemalloc when ALLOC_NO_WATERMARKS was used

__alloc_pages_slowpath() is called when the number of free pages is below
the low watermark.  If the caller is entitled to use ALLOC_NO_WATERMARKS
then the page will be marked page->pfmemalloc.  This protects more pages
than are strictly necessary as we only need to protect pages allocated
below the min watermark (the pfmemalloc reserves).

This patch only sets page->pfmemalloc when ALLOC_NO_WATERMARKS was
required to allocate the page.

[[email protected]: David noticed the problem during review]
Signed-off-by: Mel Gorman <[email protected]>
Cc: David Miller <[email protected]>
Cc: Neil Brown <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Mike Christie <[email protected]>
Cc: Eric B Munson <[email protected]>
Cc: Eric Dumazet <[email protected]>
Cc: Sebastian Andrzej Siewior <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: Christoph Lameter <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index cd5390f2f18d5315f193630352cc398bb7b97e24..f9d925451bfd31d6e6382997f5eb10789ef8da1a 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2116,8 +2116,8 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 
                page = get_page_from_freelist(gfp_mask, nodemask,
                                order, zonelist, high_zoneidx,
-                               alloc_flags, preferred_zone,
-                               migratetype);
+                               alloc_flags & ~ALLOC_NO_WATERMARKS,
+                               preferred_zone, migratetype);
                if (page) {
                        preferred_zone->compact_considered = 0;
                        preferred_zone->compact_defer_shift = 0;
@@ -2209,8 +2209,8 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
 retry:
        page = get_page_from_freelist(gfp_mask, nodemask, order,
                                        zonelist, high_zoneidx,
-                                       alloc_flags, preferred_zone,
-                                       migratetype);
+                                       alloc_flags & ~ALLOC_NO_WATERMARKS,
+                                       preferred_zone, migratetype);
 
        /*
         * If an allocation failed after direct reclaim, it could be because
@@ -2381,8 +2381,17 @@ rebalance:
                page = __alloc_pages_high_priority(gfp_mask, order,
                                zonelist, high_zoneidx, nodemask,
                                preferred_zone, migratetype);
-               if (page)
+               if (page) {
+                       /*
+                        * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was
+                        * necessary to allocate the page. The expectation is
+                        * that the caller is taking steps that will free more
+                        * memory. The caller should avoid the page being used
+                        * for !PFMEMALLOC purposes.
+                        */
+                       page->pfmemalloc = true;
                        goto got_pg;
+               }
        }
 
        /* Atomic allocations - we can't balance anything */
@@ -2499,14 +2508,6 @@ nopage:
        warn_alloc_failed(gfp_mask, order, NULL);
        return page;
 got_pg:
-       /*
-        * page->pfmemalloc is set when the caller had PFMEMALLOC set, is
-        * been OOM killed or specified __GFP_MEMALLOC. The expectation is
-        * that the caller is taking steps that will free more memory. The
-        * caller should avoid the page being used for !PFMEMALLOC purposes.
-        */
-       page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS);
-
        if (kmemcheck_enabled)
                kmemcheck_pagealloc_alloc(page, order, gfp_mask);