]> Git Repo - J-linux.git/blob - mm/compaction.c
Merge tag 'vfs-6.13-rc7.fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs
[J-linux.git] / mm / compaction.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * linux/mm/compaction.c
4  *
5  * Memory compaction for the reduction of external fragmentation. Note that
6  * this heavily depends upon page migration to do all the real heavy
7  * lifting
8  *
9  * Copyright IBM Corp. 2007-2010 Mel Gorman <[email protected]>
10  */
11 #include <linux/cpu.h>
12 #include <linux/swap.h>
13 #include <linux/migrate.h>
14 #include <linux/compaction.h>
15 #include <linux/mm_inline.h>
16 #include <linux/sched/signal.h>
17 #include <linux/backing-dev.h>
18 #include <linux/sysctl.h>
19 #include <linux/sysfs.h>
20 #include <linux/page-isolation.h>
21 #include <linux/kasan.h>
22 #include <linux/kthread.h>
23 #include <linux/freezer.h>
24 #include <linux/page_owner.h>
25 #include <linux/psi.h>
26 #include <linux/cpuset.h>
27 #include "internal.h"
28
29 #ifdef CONFIG_COMPACTION
30 /*
31  * Fragmentation score check interval for proactive compaction purposes.
32  */
33 #define HPAGE_FRAG_CHECK_INTERVAL_MSEC  (500)
34
35 static inline void count_compact_event(enum vm_event_item item)
36 {
37         count_vm_event(item);
38 }
39
40 static inline void count_compact_events(enum vm_event_item item, long delta)
41 {
42         count_vm_events(item, delta);
43 }
44
45 /*
46  * order == -1 is expected when compacting proactively via
47  * 1. /proc/sys/vm/compact_memory
48  * 2. /sys/devices/system/node/nodex/compact
49  * 3. /proc/sys/vm/compaction_proactiveness
50  */
51 static inline bool is_via_compact_memory(int order)
52 {
53         return order == -1;
54 }
55
56 #else
57 #define count_compact_event(item) do { } while (0)
58 #define count_compact_events(item, delta) do { } while (0)
59 static inline bool is_via_compact_memory(int order) { return false; }
60 #endif
61
62 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
63
64 #define CREATE_TRACE_POINTS
65 #include <trace/events/compaction.h>
66
67 #define block_start_pfn(pfn, order)     round_down(pfn, 1UL << (order))
68 #define block_end_pfn(pfn, order)       ALIGN((pfn) + 1, 1UL << (order))
69
70 /*
71  * Page order with-respect-to which proactive compaction
72  * calculates external fragmentation, which is used as
73  * the "fragmentation score" of a node/zone.
74  */
75 #if defined CONFIG_TRANSPARENT_HUGEPAGE
76 #define COMPACTION_HPAGE_ORDER  HPAGE_PMD_ORDER
77 #elif defined CONFIG_HUGETLBFS
78 #define COMPACTION_HPAGE_ORDER  HUGETLB_PAGE_ORDER
79 #else
80 #define COMPACTION_HPAGE_ORDER  (PMD_SHIFT - PAGE_SHIFT)
81 #endif
82
83 static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags)
84 {
85         post_alloc_hook(page, order, __GFP_MOVABLE);
86         return page;
87 }
88 #define mark_allocated(...)     alloc_hooks(mark_allocated_noprof(__VA_ARGS__))
89
90 static unsigned long release_free_list(struct list_head *freepages)
91 {
92         int order;
93         unsigned long high_pfn = 0;
94
95         for (order = 0; order < NR_PAGE_ORDERS; order++) {
96                 struct page *page, *next;
97
98                 list_for_each_entry_safe(page, next, &freepages[order], lru) {
99                         unsigned long pfn = page_to_pfn(page);
100
101                         list_del(&page->lru);
102                         /*
103                          * Convert free pages into post allocation pages, so
104                          * that we can free them via __free_page.
105                          */
106                         mark_allocated(page, order, __GFP_MOVABLE);
107                         __free_pages(page, order);
108                         if (pfn > high_pfn)
109                                 high_pfn = pfn;
110                 }
111         }
112         return high_pfn;
113 }
114
115 #ifdef CONFIG_COMPACTION
116 bool PageMovable(struct page *page)
117 {
118         const struct movable_operations *mops;
119
120         VM_BUG_ON_PAGE(!PageLocked(page), page);
121         if (!__PageMovable(page))
122                 return false;
123
124         mops = page_movable_ops(page);
125         if (mops)
126                 return true;
127
128         return false;
129 }
130
131 void __SetPageMovable(struct page *page, const struct movable_operations *mops)
132 {
133         VM_BUG_ON_PAGE(!PageLocked(page), page);
134         VM_BUG_ON_PAGE((unsigned long)mops & PAGE_MAPPING_MOVABLE, page);
135         page->mapping = (void *)((unsigned long)mops | PAGE_MAPPING_MOVABLE);
136 }
137 EXPORT_SYMBOL(__SetPageMovable);
138
139 void __ClearPageMovable(struct page *page)
140 {
141         VM_BUG_ON_PAGE(!PageMovable(page), page);
142         /*
143          * This page still has the type of a movable page, but it's
144          * actually not movable any more.
145          */
146         page->mapping = (void *)PAGE_MAPPING_MOVABLE;
147 }
148 EXPORT_SYMBOL(__ClearPageMovable);
149
150 /* Do not skip compaction more than 64 times */
151 #define COMPACT_MAX_DEFER_SHIFT 6
152
153 /*
154  * Compaction is deferred when compaction fails to result in a page
155  * allocation success. 1 << compact_defer_shift, compactions are skipped up
156  * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
157  */
158 static void defer_compaction(struct zone *zone, int order)
159 {
160         zone->compact_considered = 0;
161         zone->compact_defer_shift++;
162
163         if (order < zone->compact_order_failed)
164                 zone->compact_order_failed = order;
165
166         if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
167                 zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
168
169         trace_mm_compaction_defer_compaction(zone, order);
170 }
171
172 /* Returns true if compaction should be skipped this time */
173 static bool compaction_deferred(struct zone *zone, int order)
174 {
175         unsigned long defer_limit = 1UL << zone->compact_defer_shift;
176
177         if (order < zone->compact_order_failed)
178                 return false;
179
180         /* Avoid possible overflow */
181         if (++zone->compact_considered >= defer_limit) {
182                 zone->compact_considered = defer_limit;
183                 return false;
184         }
185
186         trace_mm_compaction_deferred(zone, order);
187
188         return true;
189 }
190
191 /*
192  * Update defer tracking counters after successful compaction of given order,
193  * which means an allocation either succeeded (alloc_success == true) or is
194  * expected to succeed.
195  */
196 void compaction_defer_reset(struct zone *zone, int order,
197                 bool alloc_success)
198 {
199         if (alloc_success) {
200                 zone->compact_considered = 0;
201                 zone->compact_defer_shift = 0;
202         }
203         if (order >= zone->compact_order_failed)
204                 zone->compact_order_failed = order + 1;
205
206         trace_mm_compaction_defer_reset(zone, order);
207 }
208
209 /* Returns true if restarting compaction after many failures */
210 static bool compaction_restarting(struct zone *zone, int order)
211 {
212         if (order < zone->compact_order_failed)
213                 return false;
214
215         return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
216                 zone->compact_considered >= 1UL << zone->compact_defer_shift;
217 }
218
219 /* Returns true if the pageblock should be scanned for pages to isolate. */
220 static inline bool isolation_suitable(struct compact_control *cc,
221                                         struct page *page)
222 {
223         if (cc->ignore_skip_hint)
224                 return true;
225
226         return !get_pageblock_skip(page);
227 }
228
229 static void reset_cached_positions(struct zone *zone)
230 {
231         zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
232         zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
233         zone->compact_cached_free_pfn =
234                                 pageblock_start_pfn(zone_end_pfn(zone) - 1);
235 }
236
237 #ifdef CONFIG_SPARSEMEM
238 /*
239  * If the PFN falls into an offline section, return the start PFN of the
240  * next online section. If the PFN falls into an online section or if
241  * there is no next online section, return 0.
242  */
243 static unsigned long skip_offline_sections(unsigned long start_pfn)
244 {
245         unsigned long start_nr = pfn_to_section_nr(start_pfn);
246
247         if (online_section_nr(start_nr))
248                 return 0;
249
250         while (++start_nr <= __highest_present_section_nr) {
251                 if (online_section_nr(start_nr))
252                         return section_nr_to_pfn(start_nr);
253         }
254
255         return 0;
256 }
257
258 /*
259  * If the PFN falls into an offline section, return the end PFN of the
260  * next online section in reverse. If the PFN falls into an online section
261  * or if there is no next online section in reverse, return 0.
262  */
263 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
264 {
265         unsigned long start_nr = pfn_to_section_nr(start_pfn);
266
267         if (!start_nr || online_section_nr(start_nr))
268                 return 0;
269
270         while (start_nr-- > 0) {
271                 if (online_section_nr(start_nr))
272                         return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION;
273         }
274
275         return 0;
276 }
277 #else
278 static unsigned long skip_offline_sections(unsigned long start_pfn)
279 {
280         return 0;
281 }
282
283 static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
284 {
285         return 0;
286 }
287 #endif
288
289 /*
290  * Compound pages of >= pageblock_order should consistently be skipped until
291  * released. It is always pointless to compact pages of such order (if they are
292  * migratable), and the pageblocks they occupy cannot contain any free pages.
293  */
294 static bool pageblock_skip_persistent(struct page *page)
295 {
296         if (!PageCompound(page))
297                 return false;
298
299         page = compound_head(page);
300
301         if (compound_order(page) >= pageblock_order)
302                 return true;
303
304         return false;
305 }
306
307 static bool
308 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
309                                                         bool check_target)
310 {
311         struct page *page = pfn_to_online_page(pfn);
312         struct page *block_page;
313         struct page *end_page;
314         unsigned long block_pfn;
315
316         if (!page)
317                 return false;
318         if (zone != page_zone(page))
319                 return false;
320         if (pageblock_skip_persistent(page))
321                 return false;
322
323         /*
324          * If skip is already cleared do no further checking once the
325          * restart points have been set.
326          */
327         if (check_source && check_target && !get_pageblock_skip(page))
328                 return true;
329
330         /*
331          * If clearing skip for the target scanner, do not select a
332          * non-movable pageblock as the starting point.
333          */
334         if (!check_source && check_target &&
335             get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
336                 return false;
337
338         /* Ensure the start of the pageblock or zone is online and valid */
339         block_pfn = pageblock_start_pfn(pfn);
340         block_pfn = max(block_pfn, zone->zone_start_pfn);
341         block_page = pfn_to_online_page(block_pfn);
342         if (block_page) {
343                 page = block_page;
344                 pfn = block_pfn;
345         }
346
347         /* Ensure the end of the pageblock or zone is online and valid */
348         block_pfn = pageblock_end_pfn(pfn) - 1;
349         block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
350         end_page = pfn_to_online_page(block_pfn);
351         if (!end_page)
352                 return false;
353
354         /*
355          * Only clear the hint if a sample indicates there is either a
356          * free page or an LRU page in the block. One or other condition
357          * is necessary for the block to be a migration source/target.
358          */
359         do {
360                 if (check_source && PageLRU(page)) {
361                         clear_pageblock_skip(page);
362                         return true;
363                 }
364
365                 if (check_target && PageBuddy(page)) {
366                         clear_pageblock_skip(page);
367                         return true;
368                 }
369
370                 page += (1 << PAGE_ALLOC_COSTLY_ORDER);
371         } while (page <= end_page);
372
373         return false;
374 }
375
376 /*
377  * This function is called to clear all cached information on pageblocks that
378  * should be skipped for page isolation when the migrate and free page scanner
379  * meet.
380  */
381 static void __reset_isolation_suitable(struct zone *zone)
382 {
383         unsigned long migrate_pfn = zone->zone_start_pfn;
384         unsigned long free_pfn = zone_end_pfn(zone) - 1;
385         unsigned long reset_migrate = free_pfn;
386         unsigned long reset_free = migrate_pfn;
387         bool source_set = false;
388         bool free_set = false;
389
390         /* Only flush if a full compaction finished recently */
391         if (!zone->compact_blockskip_flush)
392                 return;
393
394         zone->compact_blockskip_flush = false;
395
396         /*
397          * Walk the zone and update pageblock skip information. Source looks
398          * for PageLRU while target looks for PageBuddy. When the scanner
399          * is found, both PageBuddy and PageLRU are checked as the pageblock
400          * is suitable as both source and target.
401          */
402         for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
403                                         free_pfn -= pageblock_nr_pages) {
404                 cond_resched();
405
406                 /* Update the migrate PFN */
407                 if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
408                     migrate_pfn < reset_migrate) {
409                         source_set = true;
410                         reset_migrate = migrate_pfn;
411                         zone->compact_init_migrate_pfn = reset_migrate;
412                         zone->compact_cached_migrate_pfn[0] = reset_migrate;
413                         zone->compact_cached_migrate_pfn[1] = reset_migrate;
414                 }
415
416                 /* Update the free PFN */
417                 if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
418                     free_pfn > reset_free) {
419                         free_set = true;
420                         reset_free = free_pfn;
421                         zone->compact_init_free_pfn = reset_free;
422                         zone->compact_cached_free_pfn = reset_free;
423                 }
424         }
425
426         /* Leave no distance if no suitable block was reset */
427         if (reset_migrate >= reset_free) {
428                 zone->compact_cached_migrate_pfn[0] = migrate_pfn;
429                 zone->compact_cached_migrate_pfn[1] = migrate_pfn;
430                 zone->compact_cached_free_pfn = free_pfn;
431         }
432 }
433
434 void reset_isolation_suitable(pg_data_t *pgdat)
435 {
436         int zoneid;
437
438         for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
439                 struct zone *zone = &pgdat->node_zones[zoneid];
440                 if (!populated_zone(zone))
441                         continue;
442
443                 __reset_isolation_suitable(zone);
444         }
445 }
446
447 /*
448  * Sets the pageblock skip bit if it was clear. Note that this is a hint as
449  * locks are not required for read/writers. Returns true if it was already set.
450  */
451 static bool test_and_set_skip(struct compact_control *cc, struct page *page)
452 {
453         bool skip;
454
455         /* Do not update if skip hint is being ignored */
456         if (cc->ignore_skip_hint)
457                 return false;
458
459         skip = get_pageblock_skip(page);
460         if (!skip && !cc->no_set_skip_hint)
461                 set_pageblock_skip(page);
462
463         return skip;
464 }
465
466 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
467 {
468         struct zone *zone = cc->zone;
469
470         /* Set for isolation rather than compaction */
471         if (cc->no_set_skip_hint)
472                 return;
473
474         pfn = pageblock_end_pfn(pfn);
475
476         /* Update where async and sync compaction should restart */
477         if (pfn > zone->compact_cached_migrate_pfn[0])
478                 zone->compact_cached_migrate_pfn[0] = pfn;
479         if (cc->mode != MIGRATE_ASYNC &&
480             pfn > zone->compact_cached_migrate_pfn[1])
481                 zone->compact_cached_migrate_pfn[1] = pfn;
482 }
483
484 /*
485  * If no pages were isolated then mark this pageblock to be skipped in the
486  * future. The information is later cleared by __reset_isolation_suitable().
487  */
488 static void update_pageblock_skip(struct compact_control *cc,
489                         struct page *page, unsigned long pfn)
490 {
491         struct zone *zone = cc->zone;
492
493         if (cc->no_set_skip_hint)
494                 return;
495
496         set_pageblock_skip(page);
497
498         if (pfn < zone->compact_cached_free_pfn)
499                 zone->compact_cached_free_pfn = pfn;
500 }
501 #else
502 static inline bool isolation_suitable(struct compact_control *cc,
503                                         struct page *page)
504 {
505         return true;
506 }
507
508 static inline bool pageblock_skip_persistent(struct page *page)
509 {
510         return false;
511 }
512
513 static inline void update_pageblock_skip(struct compact_control *cc,
514                         struct page *page, unsigned long pfn)
515 {
516 }
517
518 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
519 {
520 }
521
522 static bool test_and_set_skip(struct compact_control *cc, struct page *page)
523 {
524         return false;
525 }
526 #endif /* CONFIG_COMPACTION */
527
528 /*
529  * Compaction requires the taking of some coarse locks that are potentially
530  * very heavily contended. For async compaction, trylock and record if the
531  * lock is contended. The lock will still be acquired but compaction will
532  * abort when the current block is finished regardless of success rate.
533  * Sync compaction acquires the lock.
534  *
535  * Always returns true which makes it easier to track lock state in callers.
536  */
537 static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
538                                                 struct compact_control *cc)
539         __acquires(lock)
540 {
541         /* Track if the lock is contended in async mode */
542         if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
543                 if (spin_trylock_irqsave(lock, *flags))
544                         return true;
545
546                 cc->contended = true;
547         }
548
549         spin_lock_irqsave(lock, *flags);
550         return true;
551 }
552
553 /*
554  * Compaction requires the taking of some coarse locks that are potentially
555  * very heavily contended. The lock should be periodically unlocked to avoid
556  * having disabled IRQs for a long time, even when there is nobody waiting on
557  * the lock. It might also be that allowing the IRQs will result in
558  * need_resched() becoming true. If scheduling is needed, compaction schedules.
559  * Either compaction type will also abort if a fatal signal is pending.
560  * In either case if the lock was locked, it is dropped and not regained.
561  *
562  * Returns true if compaction should abort due to fatal signal pending.
563  * Returns false when compaction can continue.
564  */
565 static bool compact_unlock_should_abort(spinlock_t *lock,
566                 unsigned long flags, bool *locked, struct compact_control *cc)
567 {
568         if (*locked) {
569                 spin_unlock_irqrestore(lock, flags);
570                 *locked = false;
571         }
572
573         if (fatal_signal_pending(current)) {
574                 cc->contended = true;
575                 return true;
576         }
577
578         cond_resched();
579
580         return false;
581 }
582
583 /*
584  * Isolate free pages onto a private freelist. If @strict is true, will abort
585  * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
586  * (even though it may still end up isolating some pages).
587  */
588 static unsigned long isolate_freepages_block(struct compact_control *cc,
589                                 unsigned long *start_pfn,
590                                 unsigned long end_pfn,
591                                 struct list_head *freelist,
592                                 unsigned int stride,
593                                 bool strict)
594 {
595         int nr_scanned = 0, total_isolated = 0;
596         struct page *page;
597         unsigned long flags = 0;
598         bool locked = false;
599         unsigned long blockpfn = *start_pfn;
600         unsigned int order;
601
602         /* Strict mode is for isolation, speed is secondary */
603         if (strict)
604                 stride = 1;
605
606         page = pfn_to_page(blockpfn);
607
608         /* Isolate free pages. */
609         for (; blockpfn < end_pfn; blockpfn += stride, page += stride) {
610                 int isolated;
611
612                 /*
613                  * Periodically drop the lock (if held) regardless of its
614                  * contention, to give chance to IRQs. Abort if fatal signal
615                  * pending.
616                  */
617                 if (!(blockpfn % COMPACT_CLUSTER_MAX)
618                     && compact_unlock_should_abort(&cc->zone->lock, flags,
619                                                                 &locked, cc))
620                         break;
621
622                 nr_scanned++;
623
624                 /*
625                  * For compound pages such as THP and hugetlbfs, we can save
626                  * potentially a lot of iterations if we skip them at once.
627                  * The check is racy, but we can consider only valid values
628                  * and the only danger is skipping too much.
629                  */
630                 if (PageCompound(page)) {
631                         const unsigned int order = compound_order(page);
632
633                         if (blockpfn + (1UL << order) <= end_pfn) {
634                                 blockpfn += (1UL << order) - 1;
635                                 page += (1UL << order) - 1;
636                                 nr_scanned += (1UL << order) - 1;
637                         }
638
639                         goto isolate_fail;
640                 }
641
642                 if (!PageBuddy(page))
643                         goto isolate_fail;
644
645                 /* If we already hold the lock, we can skip some rechecking. */
646                 if (!locked) {
647                         locked = compact_lock_irqsave(&cc->zone->lock,
648                                                                 &flags, cc);
649
650                         /* Recheck this is a buddy page under lock */
651                         if (!PageBuddy(page))
652                                 goto isolate_fail;
653                 }
654
655                 /* Found a free page, will break it into order-0 pages */
656                 order = buddy_order(page);
657                 isolated = __isolate_free_page(page, order);
658                 if (!isolated)
659                         break;
660                 set_page_private(page, order);
661
662                 nr_scanned += isolated - 1;
663                 total_isolated += isolated;
664                 cc->nr_freepages += isolated;
665                 list_add_tail(&page->lru, &freelist[order]);
666
667                 if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
668                         blockpfn += isolated;
669                         break;
670                 }
671                 /* Advance to the end of split page */
672                 blockpfn += isolated - 1;
673                 page += isolated - 1;
674                 continue;
675
676 isolate_fail:
677                 if (strict)
678                         break;
679
680         }
681
682         if (locked)
683                 spin_unlock_irqrestore(&cc->zone->lock, flags);
684
685         /*
686          * Be careful to not go outside of the pageblock.
687          */
688         if (unlikely(blockpfn > end_pfn))
689                 blockpfn = end_pfn;
690
691         trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
692                                         nr_scanned, total_isolated);
693
694         /* Record how far we have got within the block */
695         *start_pfn = blockpfn;
696
697         /*
698          * If strict isolation is requested by CMA then check that all the
699          * pages requested were isolated. If there were any failures, 0 is
700          * returned and CMA will fail.
701          */
702         if (strict && blockpfn < end_pfn)
703                 total_isolated = 0;
704
705         cc->total_free_scanned += nr_scanned;
706         if (total_isolated)
707                 count_compact_events(COMPACTISOLATED, total_isolated);
708         return total_isolated;
709 }
710
711 /**
712  * isolate_freepages_range() - isolate free pages.
713  * @cc:        Compaction control structure.
714  * @start_pfn: The first PFN to start isolating.
715  * @end_pfn:   The one-past-last PFN.
716  *
717  * Non-free pages, invalid PFNs, or zone boundaries within the
718  * [start_pfn, end_pfn) range are considered errors, cause function to
719  * undo its actions and return zero. cc->freepages[] are empty.
720  *
721  * Otherwise, function returns one-past-the-last PFN of isolated page
722  * (which may be greater then end_pfn if end fell in a middle of
723  * a free page). cc->freepages[] contain free pages isolated.
724  */
725 unsigned long
726 isolate_freepages_range(struct compact_control *cc,
727                         unsigned long start_pfn, unsigned long end_pfn)
728 {
729         unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
730         int order;
731
732         for (order = 0; order < NR_PAGE_ORDERS; order++)
733                 INIT_LIST_HEAD(&cc->freepages[order]);
734
735         pfn = start_pfn;
736         block_start_pfn = pageblock_start_pfn(pfn);
737         if (block_start_pfn < cc->zone->zone_start_pfn)
738                 block_start_pfn = cc->zone->zone_start_pfn;
739         block_end_pfn = pageblock_end_pfn(pfn);
740
741         for (; pfn < end_pfn; pfn += isolated,
742                                 block_start_pfn = block_end_pfn,
743                                 block_end_pfn += pageblock_nr_pages) {
744                 /* Protect pfn from changing by isolate_freepages_block */
745                 unsigned long isolate_start_pfn = pfn;
746
747                 /*
748                  * pfn could pass the block_end_pfn if isolated freepage
749                  * is more than pageblock order. In this case, we adjust
750                  * scanning range to right one.
751                  */
752                 if (pfn >= block_end_pfn) {
753                         block_start_pfn = pageblock_start_pfn(pfn);
754                         block_end_pfn = pageblock_end_pfn(pfn);
755                 }
756
757                 block_end_pfn = min(block_end_pfn, end_pfn);
758
759                 if (!pageblock_pfn_to_page(block_start_pfn,
760                                         block_end_pfn, cc->zone))
761                         break;
762
763                 isolated = isolate_freepages_block(cc, &isolate_start_pfn,
764                                         block_end_pfn, cc->freepages, 0, true);
765
766                 /*
767                  * In strict mode, isolate_freepages_block() returns 0 if
768                  * there are any holes in the block (ie. invalid PFNs or
769                  * non-free pages).
770                  */
771                 if (!isolated)
772                         break;
773
774                 /*
775                  * If we managed to isolate pages, it is always (1 << n) *
776                  * pageblock_nr_pages for some non-negative n.  (Max order
777                  * page may span two pageblocks).
778                  */
779         }
780
781         if (pfn < end_pfn) {
782                 /* Loop terminated early, cleanup. */
783                 release_free_list(cc->freepages);
784                 return 0;
785         }
786
787         /* We don't use freelists for anything. */
788         return pfn;
789 }
790
791 /* Similar to reclaim, but different enough that they don't share logic */
792 static bool too_many_isolated(struct compact_control *cc)
793 {
794         pg_data_t *pgdat = cc->zone->zone_pgdat;
795         bool too_many;
796
797         unsigned long active, inactive, isolated;
798
799         inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
800                         node_page_state(pgdat, NR_INACTIVE_ANON);
801         active = node_page_state(pgdat, NR_ACTIVE_FILE) +
802                         node_page_state(pgdat, NR_ACTIVE_ANON);
803         isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
804                         node_page_state(pgdat, NR_ISOLATED_ANON);
805
806         /*
807          * Allow GFP_NOFS to isolate past the limit set for regular
808          * compaction runs. This prevents an ABBA deadlock when other
809          * compactors have already isolated to the limit, but are
810          * blocked on filesystem locks held by the GFP_NOFS thread.
811          */
812         if (cc->gfp_mask & __GFP_FS) {
813                 inactive >>= 3;
814                 active >>= 3;
815         }
816
817         too_many = isolated > (inactive + active) / 2;
818         if (!too_many)
819                 wake_throttle_isolated(pgdat);
820
821         return too_many;
822 }
823
824 /**
825  * skip_isolation_on_order() - determine when to skip folio isolation based on
826  *                             folio order and compaction target order
827  * @order:              to-be-isolated folio order
828  * @target_order:       compaction target order
829  *
830  * This avoids unnecessary folio isolations during compaction.
831  */
832 static bool skip_isolation_on_order(int order, int target_order)
833 {
834         /*
835          * Unless we are performing global compaction (i.e.,
836          * is_via_compact_memory), skip any folios that are larger than the
837          * target order: we wouldn't be here if we'd have a free folio with
838          * the desired target_order, so migrating this folio would likely fail
839          * later.
840          */
841         if (!is_via_compact_memory(target_order) && order >= target_order)
842                 return true;
843         /*
844          * We limit memory compaction to pageblocks and won't try
845          * creating free blocks of memory that are larger than that.
846          */
847         return order >= pageblock_order;
848 }
849
850 /**
851  * isolate_migratepages_block() - isolate all migrate-able pages within
852  *                                a single pageblock
853  * @cc:         Compaction control structure.
854  * @low_pfn:    The first PFN to isolate
855  * @end_pfn:    The one-past-the-last PFN to isolate, within same pageblock
856  * @mode:       Isolation mode to be used.
857  *
858  * Isolate all pages that can be migrated from the range specified by
859  * [low_pfn, end_pfn). The range is expected to be within same pageblock.
860  * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,
861  * -ENOMEM in case we could not allocate a page, or 0.
862  * cc->migrate_pfn will contain the next pfn to scan.
863  *
864  * The pages are isolated on cc->migratepages list (not required to be empty),
865  * and cc->nr_migratepages is updated accordingly.
866  */
867 static int
868 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
869                         unsigned long end_pfn, isolate_mode_t mode)
870 {
871         pg_data_t *pgdat = cc->zone->zone_pgdat;
872         unsigned long nr_scanned = 0, nr_isolated = 0;
873         struct lruvec *lruvec;
874         unsigned long flags = 0;
875         struct lruvec *locked = NULL;
876         struct folio *folio = NULL;
877         struct page *page = NULL, *valid_page = NULL;
878         struct address_space *mapping;
879         unsigned long start_pfn = low_pfn;
880         bool skip_on_failure = false;
881         unsigned long next_skip_pfn = 0;
882         bool skip_updated = false;
883         int ret = 0;
884
885         cc->migrate_pfn = low_pfn;
886
887         /*
888          * Ensure that there are not too many pages isolated from the LRU
889          * list by either parallel reclaimers or compaction. If there are,
890          * delay for some time until fewer pages are isolated
891          */
892         while (unlikely(too_many_isolated(cc))) {
893                 /* stop isolation if there are still pages not migrated */
894                 if (cc->nr_migratepages)
895                         return -EAGAIN;
896
897                 /* async migration should just abort */
898                 if (cc->mode == MIGRATE_ASYNC)
899                         return -EAGAIN;
900
901                 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
902
903                 if (fatal_signal_pending(current))
904                         return -EINTR;
905         }
906
907         cond_resched();
908
909         if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
910                 skip_on_failure = true;
911                 next_skip_pfn = block_end_pfn(low_pfn, cc->order);
912         }
913
914         /* Time to isolate some pages for migration */
915         for (; low_pfn < end_pfn; low_pfn++) {
916                 bool is_dirty, is_unevictable;
917
918                 if (skip_on_failure && low_pfn >= next_skip_pfn) {
919                         /*
920                          * We have isolated all migration candidates in the
921                          * previous order-aligned block, and did not skip it due
922                          * to failure. We should migrate the pages now and
923                          * hopefully succeed compaction.
924                          */
925                         if (nr_isolated)
926                                 break;
927
928                         /*
929                          * We failed to isolate in the previous order-aligned
930                          * block. Set the new boundary to the end of the
931                          * current block. Note we can't simply increase
932                          * next_skip_pfn by 1 << order, as low_pfn might have
933                          * been incremented by a higher number due to skipping
934                          * a compound or a high-order buddy page in the
935                          * previous loop iteration.
936                          */
937                         next_skip_pfn = block_end_pfn(low_pfn, cc->order);
938                 }
939
940                 /*
941                  * Periodically drop the lock (if held) regardless of its
942                  * contention, to give chance to IRQs. Abort completely if
943                  * a fatal signal is pending.
944                  */
945                 if (!(low_pfn % COMPACT_CLUSTER_MAX)) {
946                         if (locked) {
947                                 unlock_page_lruvec_irqrestore(locked, flags);
948                                 locked = NULL;
949                         }
950
951                         if (fatal_signal_pending(current)) {
952                                 cc->contended = true;
953                                 ret = -EINTR;
954
955                                 goto fatal_pending;
956                         }
957
958                         cond_resched();
959                 }
960
961                 nr_scanned++;
962
963                 page = pfn_to_page(low_pfn);
964
965                 /*
966                  * Check if the pageblock has already been marked skipped.
967                  * Only the first PFN is checked as the caller isolates
968                  * COMPACT_CLUSTER_MAX at a time so the second call must
969                  * not falsely conclude that the block should be skipped.
970                  */
971                 if (!valid_page && (pageblock_aligned(low_pfn) ||
972                                     low_pfn == cc->zone->zone_start_pfn)) {
973                         if (!isolation_suitable(cc, page)) {
974                                 low_pfn = end_pfn;
975                                 folio = NULL;
976                                 goto isolate_abort;
977                         }
978                         valid_page = page;
979                 }
980
981                 if (PageHuge(page)) {
982                         /*
983                          * skip hugetlbfs if we are not compacting for pages
984                          * bigger than its order. THPs and other compound pages
985                          * are handled below.
986                          */
987                         if (!cc->alloc_contig) {
988                                 const unsigned int order = compound_order(page);
989
990                                 if (order <= MAX_PAGE_ORDER) {
991                                         low_pfn += (1UL << order) - 1;
992                                         nr_scanned += (1UL << order) - 1;
993                                 }
994                                 goto isolate_fail;
995                         }
996                         /* for alloc_contig case */
997                         if (locked) {
998                                 unlock_page_lruvec_irqrestore(locked, flags);
999                                 locked = NULL;
1000                         }
1001
1002                         ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
1003
1004                         /*
1005                          * Fail isolation in case isolate_or_dissolve_huge_page()
1006                          * reports an error. In case of -ENOMEM, abort right away.
1007                          */
1008                         if (ret < 0) {
1009                                  /* Do not report -EBUSY down the chain */
1010                                 if (ret == -EBUSY)
1011                                         ret = 0;
1012                                 low_pfn += compound_nr(page) - 1;
1013                                 nr_scanned += compound_nr(page) - 1;
1014                                 goto isolate_fail;
1015                         }
1016
1017                         if (PageHuge(page)) {
1018                                 /*
1019                                  * Hugepage was successfully isolated and placed
1020                                  * on the cc->migratepages list.
1021                                  */
1022                                 folio = page_folio(page);
1023                                 low_pfn += folio_nr_pages(folio) - 1;
1024                                 goto isolate_success_no_list;
1025                         }
1026
1027                         /*
1028                          * Ok, the hugepage was dissolved. Now these pages are
1029                          * Buddy and cannot be re-allocated because they are
1030                          * isolated. Fall-through as the check below handles
1031                          * Buddy pages.
1032                          */
1033                 }
1034
1035                 /*
1036                  * Skip if free. We read page order here without zone lock
1037                  * which is generally unsafe, but the race window is small and
1038                  * the worst thing that can happen is that we skip some
1039                  * potential isolation targets.
1040                  */
1041                 if (PageBuddy(page)) {
1042                         unsigned long freepage_order = buddy_order_unsafe(page);
1043
1044                         /*
1045                          * Without lock, we cannot be sure that what we got is
1046                          * a valid page order. Consider only values in the
1047                          * valid order range to prevent low_pfn overflow.
1048                          */
1049                         if (freepage_order > 0 && freepage_order <= MAX_PAGE_ORDER) {
1050                                 low_pfn += (1UL << freepage_order) - 1;
1051                                 nr_scanned += (1UL << freepage_order) - 1;
1052                         }
1053                         continue;
1054                 }
1055
1056                 /*
1057                  * Regardless of being on LRU, compound pages such as THP
1058                  * (hugetlbfs is handled above) are not to be compacted unless
1059                  * we are attempting an allocation larger than the compound
1060                  * page size. We can potentially save a lot of iterations if we
1061                  * skip them at once. The check is racy, but we can consider
1062                  * only valid values and the only danger is skipping too much.
1063                  */
1064                 if (PageCompound(page) && !cc->alloc_contig) {
1065                         const unsigned int order = compound_order(page);
1066
1067                         /* Skip based on page order and compaction target order. */
1068                         if (skip_isolation_on_order(order, cc->order)) {
1069                                 if (order <= MAX_PAGE_ORDER) {
1070                                         low_pfn += (1UL << order) - 1;
1071                                         nr_scanned += (1UL << order) - 1;
1072                                 }
1073                                 goto isolate_fail;
1074                         }
1075                 }
1076
1077                 /*
1078                  * Check may be lockless but that's ok as we recheck later.
1079                  * It's possible to migrate LRU and non-lru movable pages.
1080                  * Skip any other type of page
1081                  */
1082                 if (!PageLRU(page)) {
1083                         /*
1084                          * __PageMovable can return false positive so we need
1085                          * to verify it under page_lock.
1086                          */
1087                         if (unlikely(__PageMovable(page)) &&
1088                                         !PageIsolated(page)) {
1089                                 if (locked) {
1090                                         unlock_page_lruvec_irqrestore(locked, flags);
1091                                         locked = NULL;
1092                                 }
1093
1094                                 if (isolate_movable_page(page, mode)) {
1095                                         folio = page_folio(page);
1096                                         goto isolate_success;
1097                                 }
1098                         }
1099
1100                         goto isolate_fail;
1101                 }
1102
1103                 /*
1104                  * Be careful not to clear PageLRU until after we're
1105                  * sure the page is not being freed elsewhere -- the
1106                  * page release code relies on it.
1107                  */
1108                 folio = folio_get_nontail_page(page);
1109                 if (unlikely(!folio))
1110                         goto isolate_fail;
1111
1112                 /*
1113                  * Migration will fail if an anonymous page is pinned in memory,
1114                  * so avoid taking lru_lock and isolating it unnecessarily in an
1115                  * admittedly racy check.
1116                  */
1117                 mapping = folio_mapping(folio);
1118                 if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))
1119                         goto isolate_fail_put;
1120
1121                 /*
1122                  * Only allow to migrate anonymous pages in GFP_NOFS context
1123                  * because those do not depend on fs locks.
1124                  */
1125                 if (!(cc->gfp_mask & __GFP_FS) && mapping)
1126                         goto isolate_fail_put;
1127
1128                 /* Only take pages on LRU: a check now makes later tests safe */
1129                 if (!folio_test_lru(folio))
1130                         goto isolate_fail_put;
1131
1132                 is_unevictable = folio_test_unevictable(folio);
1133
1134                 /* Compaction might skip unevictable pages but CMA takes them */
1135                 if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
1136                         goto isolate_fail_put;
1137
1138                 /*
1139                  * To minimise LRU disruption, the caller can indicate with
1140                  * ISOLATE_ASYNC_MIGRATE that it only wants to isolate pages
1141                  * it will be able to migrate without blocking - clean pages
1142                  * for the most part.  PageWriteback would require blocking.
1143                  */
1144                 if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
1145                         goto isolate_fail_put;
1146
1147                 is_dirty = folio_test_dirty(folio);
1148
1149                 if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) ||
1150                     (mapping && is_unevictable)) {
1151                         bool migrate_dirty = true;
1152                         bool is_inaccessible;
1153
1154                         /*
1155                          * Only folios without mappings or that have
1156                          * a ->migrate_folio callback are possible to migrate
1157                          * without blocking.
1158                          *
1159                          * Folios from inaccessible mappings are not migratable.
1160                          *
1161                          * However, we can be racing with truncation, which can
1162                          * free the mapping that we need to check. Truncation
1163                          * holds the folio lock until after the folio is removed
1164                          * from the page so holding it ourselves is sufficient.
1165                          *
1166                          * To avoid locking the folio just to check inaccessible,
1167                          * assume every inaccessible folio is also unevictable,
1168                          * which is a cheaper test.  If our assumption goes
1169                          * wrong, it's not a correctness bug, just potentially
1170                          * wasted cycles.
1171                          */
1172                         if (!folio_trylock(folio))
1173                                 goto isolate_fail_put;
1174
1175                         mapping = folio_mapping(folio);
1176                         if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) {
1177                                 migrate_dirty = !mapping ||
1178                                                 mapping->a_ops->migrate_folio;
1179                         }
1180                         is_inaccessible = mapping && mapping_inaccessible(mapping);
1181                         folio_unlock(folio);
1182                         if (!migrate_dirty || is_inaccessible)
1183                                 goto isolate_fail_put;
1184                 }
1185
1186                 /* Try isolate the folio */
1187                 if (!folio_test_clear_lru(folio))
1188                         goto isolate_fail_put;
1189
1190                 lruvec = folio_lruvec(folio);
1191
1192                 /* If we already hold the lock, we can skip some rechecking */
1193                 if (lruvec != locked) {
1194                         if (locked)
1195                                 unlock_page_lruvec_irqrestore(locked, flags);
1196
1197                         compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
1198                         locked = lruvec;
1199
1200                         lruvec_memcg_debug(lruvec, folio);
1201
1202                         /*
1203                          * Try get exclusive access under lock. If marked for
1204                          * skip, the scan is aborted unless the current context
1205                          * is a rescan to reach the end of the pageblock.
1206                          */
1207                         if (!skip_updated && valid_page) {
1208                                 skip_updated = true;
1209                                 if (test_and_set_skip(cc, valid_page) &&
1210                                     !cc->finish_pageblock) {
1211                                         low_pfn = end_pfn;
1212                                         goto isolate_abort;
1213                                 }
1214                         }
1215
1216                         /*
1217                          * Check LRU folio order under the lock
1218                          */
1219                         if (unlikely(skip_isolation_on_order(folio_order(folio),
1220                                                              cc->order) &&
1221                                      !cc->alloc_contig)) {
1222                                 low_pfn += folio_nr_pages(folio) - 1;
1223                                 nr_scanned += folio_nr_pages(folio) - 1;
1224                                 folio_set_lru(folio);
1225                                 goto isolate_fail_put;
1226                         }
1227                 }
1228
1229                 /* The folio is taken off the LRU */
1230                 if (folio_test_large(folio))
1231                         low_pfn += folio_nr_pages(folio) - 1;
1232
1233                 /* Successfully isolated */
1234                 lruvec_del_folio(lruvec, folio);
1235                 node_stat_mod_folio(folio,
1236                                 NR_ISOLATED_ANON + folio_is_file_lru(folio),
1237                                 folio_nr_pages(folio));
1238
1239 isolate_success:
1240                 list_add(&folio->lru, &cc->migratepages);
1241 isolate_success_no_list:
1242                 cc->nr_migratepages += folio_nr_pages(folio);
1243                 nr_isolated += folio_nr_pages(folio);
1244                 nr_scanned += folio_nr_pages(folio) - 1;
1245
1246                 /*
1247                  * Avoid isolating too much unless this block is being
1248                  * fully scanned (e.g. dirty/writeback pages, parallel allocation)
1249                  * or a lock is contended. For contention, isolate quickly to
1250                  * potentially remove one source of contention.
1251                  */
1252                 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX &&
1253                     !cc->finish_pageblock && !cc->contended) {
1254                         ++low_pfn;
1255                         break;
1256                 }
1257
1258                 continue;
1259
1260 isolate_fail_put:
1261                 /* Avoid potential deadlock in freeing page under lru_lock */
1262                 if (locked) {
1263                         unlock_page_lruvec_irqrestore(locked, flags);
1264                         locked = NULL;
1265                 }
1266                 folio_put(folio);
1267
1268 isolate_fail:
1269                 if (!skip_on_failure && ret != -ENOMEM)
1270                         continue;
1271
1272                 /*
1273                  * We have isolated some pages, but then failed. Release them
1274                  * instead of migrating, as we cannot form the cc->order buddy
1275                  * page anyway.
1276                  */
1277                 if (nr_isolated) {
1278                         if (locked) {
1279                                 unlock_page_lruvec_irqrestore(locked, flags);
1280                                 locked = NULL;
1281                         }
1282                         putback_movable_pages(&cc->migratepages);
1283                         cc->nr_migratepages = 0;
1284                         nr_isolated = 0;
1285                 }
1286
1287                 if (low_pfn < next_skip_pfn) {
1288                         low_pfn = next_skip_pfn - 1;
1289                         /*
1290                          * The check near the loop beginning would have updated
1291                          * next_skip_pfn too, but this is a bit simpler.
1292                          */
1293                         next_skip_pfn += 1UL << cc->order;
1294                 }
1295
1296                 if (ret == -ENOMEM)
1297                         break;
1298         }
1299
1300         /*
1301          * The PageBuddy() check could have potentially brought us outside
1302          * the range to be scanned.
1303          */
1304         if (unlikely(low_pfn > end_pfn))
1305                 low_pfn = end_pfn;
1306
1307         folio = NULL;
1308
1309 isolate_abort:
1310         if (locked)
1311                 unlock_page_lruvec_irqrestore(locked, flags);
1312         if (folio) {
1313                 folio_set_lru(folio);
1314                 folio_put(folio);
1315         }
1316
1317         /*
1318          * Update the cached scanner pfn once the pageblock has been scanned.
1319          * Pages will either be migrated in which case there is no point
1320          * scanning in the near future or migration failed in which case the
1321          * failure reason may persist. The block is marked for skipping if
1322          * there were no pages isolated in the block or if the block is
1323          * rescanned twice in a row.
1324          */
1325         if (low_pfn == end_pfn && (!nr_isolated || cc->finish_pageblock)) {
1326                 if (!cc->no_set_skip_hint && valid_page && !skip_updated)
1327                         set_pageblock_skip(valid_page);
1328                 update_cached_migrate(cc, low_pfn);
1329         }
1330
1331         trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1332                                                 nr_scanned, nr_isolated);
1333
1334 fatal_pending:
1335         cc->total_migrate_scanned += nr_scanned;
1336         if (nr_isolated)
1337                 count_compact_events(COMPACTISOLATED, nr_isolated);
1338
1339         cc->migrate_pfn = low_pfn;
1340
1341         return ret;
1342 }
1343
1344 /**
1345  * isolate_migratepages_range() - isolate migrate-able pages in a PFN range
1346  * @cc:        Compaction control structure.
1347  * @start_pfn: The first PFN to start isolating.
1348  * @end_pfn:   The one-past-last PFN.
1349  *
1350  * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM
1351  * in case we could not allocate a page, or 0.
1352  */
1353 int
1354 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1355                                                         unsigned long end_pfn)
1356 {
1357         unsigned long pfn, block_start_pfn, block_end_pfn;
1358         int ret = 0;
1359
1360         /* Scan block by block. First and last block may be incomplete */
1361         pfn = start_pfn;
1362         block_start_pfn = pageblock_start_pfn(pfn);
1363         if (block_start_pfn < cc->zone->zone_start_pfn)
1364                 block_start_pfn = cc->zone->zone_start_pfn;
1365         block_end_pfn = pageblock_end_pfn(pfn);
1366
1367         for (; pfn < end_pfn; pfn = block_end_pfn,
1368                                 block_start_pfn = block_end_pfn,
1369                                 block_end_pfn += pageblock_nr_pages) {
1370
1371                 block_end_pfn = min(block_end_pfn, end_pfn);
1372
1373                 if (!pageblock_pfn_to_page(block_start_pfn,
1374                                         block_end_pfn, cc->zone))
1375                         continue;
1376
1377                 ret = isolate_migratepages_block(cc, pfn, block_end_pfn,
1378                                                  ISOLATE_UNEVICTABLE);
1379
1380                 if (ret)
1381                         break;
1382
1383                 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX)
1384                         break;
1385         }
1386
1387         return ret;
1388 }
1389
1390 #endif /* CONFIG_COMPACTION || CONFIG_CMA */
1391 #ifdef CONFIG_COMPACTION
1392
1393 static bool suitable_migration_source(struct compact_control *cc,
1394                                                         struct page *page)
1395 {
1396         int block_mt;
1397
1398         if (pageblock_skip_persistent(page))
1399                 return false;
1400
1401         if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
1402                 return true;
1403
1404         block_mt = get_pageblock_migratetype(page);
1405
1406         if (cc->migratetype == MIGRATE_MOVABLE)
1407                 return is_migrate_movable(block_mt);
1408         else
1409                 return block_mt == cc->migratetype;
1410 }
1411
1412 /* Returns true if the page is within a block suitable for migration to */
1413 static bool suitable_migration_target(struct compact_control *cc,
1414                                                         struct page *page)
1415 {
1416         /* If the page is a large free page, then disallow migration */
1417         if (PageBuddy(page)) {
1418                 int order = cc->order > 0 ? cc->order : pageblock_order;
1419
1420                 /*
1421                  * We are checking page_order without zone->lock taken. But
1422                  * the only small danger is that we skip a potentially suitable
1423                  * pageblock, so it's not worth to check order for valid range.
1424                  */
1425                 if (buddy_order_unsafe(page) >= order)
1426                         return false;
1427         }
1428
1429         if (cc->ignore_block_suitable)
1430                 return true;
1431
1432         /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
1433         if (is_migrate_movable(get_pageblock_migratetype(page)))
1434                 return true;
1435
1436         /* Otherwise skip the block */
1437         return false;
1438 }
1439
1440 static inline unsigned int
1441 freelist_scan_limit(struct compact_control *cc)
1442 {
1443         unsigned short shift = BITS_PER_LONG - 1;
1444
1445         return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
1446 }
1447
1448 /*
1449  * Test whether the free scanner has reached the same or lower pageblock than
1450  * the migration scanner, and compaction should thus terminate.
1451  */
1452 static inline bool compact_scanners_met(struct compact_control *cc)
1453 {
1454         return (cc->free_pfn >> pageblock_order)
1455                 <= (cc->migrate_pfn >> pageblock_order);
1456 }
1457
1458 /*
1459  * Used when scanning for a suitable migration target which scans freelists
1460  * in reverse. Reorders the list such as the unscanned pages are scanned
1461  * first on the next iteration of the free scanner
1462  */
1463 static void
1464 move_freelist_head(struct list_head *freelist, struct page *freepage)
1465 {
1466         LIST_HEAD(sublist);
1467
1468         if (!list_is_first(&freepage->buddy_list, freelist)) {
1469                 list_cut_before(&sublist, freelist, &freepage->buddy_list);
1470                 list_splice_tail(&sublist, freelist);
1471         }
1472 }
1473
1474 /*
1475  * Similar to move_freelist_head except used by the migration scanner
1476  * when scanning forward. It's possible for these list operations to
1477  * move against each other if they search the free list exactly in
1478  * lockstep.
1479  */
1480 static void
1481 move_freelist_tail(struct list_head *freelist, struct page *freepage)
1482 {
1483         LIST_HEAD(sublist);
1484
1485         if (!list_is_last(&freepage->buddy_list, freelist)) {
1486                 list_cut_position(&sublist, freelist, &freepage->buddy_list);
1487                 list_splice_tail(&sublist, freelist);
1488         }
1489 }
1490
1491 static void
1492 fast_isolate_around(struct compact_control *cc, unsigned long pfn)
1493 {
1494         unsigned long start_pfn, end_pfn;
1495         struct page *page;
1496
1497         /* Do not search around if there are enough pages already */
1498         if (cc->nr_freepages >= cc->nr_migratepages)
1499                 return;
1500
1501         /* Minimise scanning during async compaction */
1502         if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
1503                 return;
1504
1505         /* Pageblock boundaries */
1506         start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);
1507         end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));
1508
1509         page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);
1510         if (!page)
1511                 return;
1512
1513         isolate_freepages_block(cc, &start_pfn, end_pfn, cc->freepages, 1, false);
1514
1515         /* Skip this pageblock in the future as it's full or nearly full */
1516         if (start_pfn == end_pfn && !cc->no_set_skip_hint)
1517                 set_pageblock_skip(page);
1518 }
1519
1520 /* Search orders in round-robin fashion */
1521 static int next_search_order(struct compact_control *cc, int order)
1522 {
1523         order--;
1524         if (order < 0)
1525                 order = cc->order - 1;
1526
1527         /* Search wrapped around? */
1528         if (order == cc->search_order) {
1529                 cc->search_order--;
1530                 if (cc->search_order < 0)
1531                         cc->search_order = cc->order - 1;
1532                 return -1;
1533         }
1534
1535         return order;
1536 }
1537
1538 static void fast_isolate_freepages(struct compact_control *cc)
1539 {
1540         unsigned int limit = max(1U, freelist_scan_limit(cc) >> 1);
1541         unsigned int nr_scanned = 0, total_isolated = 0;
1542         unsigned long low_pfn, min_pfn, highest = 0;
1543         unsigned long nr_isolated = 0;
1544         unsigned long distance;
1545         struct page *page = NULL;
1546         bool scan_start = false;
1547         int order;
1548
1549         /* Full compaction passes in a negative order */
1550         if (cc->order <= 0)
1551                 return;
1552
1553         /*
1554          * If starting the scan, use a deeper search and use the highest
1555          * PFN found if a suitable one is not found.
1556          */
1557         if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
1558                 limit = pageblock_nr_pages >> 1;
1559                 scan_start = true;
1560         }
1561
1562         /*
1563          * Preferred point is in the top quarter of the scan space but take
1564          * a pfn from the top half if the search is problematic.
1565          */
1566         distance = (cc->free_pfn - cc->migrate_pfn);
1567         low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
1568         min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
1569
1570         if (WARN_ON_ONCE(min_pfn > low_pfn))
1571                 low_pfn = min_pfn;
1572
1573         /*
1574          * Search starts from the last successful isolation order or the next
1575          * order to search after a previous failure
1576          */
1577         cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
1578
1579         for (order = cc->search_order;
1580              !page && order >= 0;
1581              order = next_search_order(cc, order)) {
1582                 struct free_area *area = &cc->zone->free_area[order];
1583                 struct list_head *freelist;
1584                 struct page *freepage;
1585                 unsigned long flags;
1586                 unsigned int order_scanned = 0;
1587                 unsigned long high_pfn = 0;
1588
1589                 if (!area->nr_free)
1590                         continue;
1591
1592                 spin_lock_irqsave(&cc->zone->lock, flags);
1593                 freelist = &area->free_list[MIGRATE_MOVABLE];
1594                 list_for_each_entry_reverse(freepage, freelist, buddy_list) {
1595                         unsigned long pfn;
1596
1597                         order_scanned++;
1598                         nr_scanned++;
1599                         pfn = page_to_pfn(freepage);
1600
1601                         if (pfn >= highest)
1602                                 highest = max(pageblock_start_pfn(pfn),
1603                                               cc->zone->zone_start_pfn);
1604
1605                         if (pfn >= low_pfn) {
1606                                 cc->fast_search_fail = 0;
1607                                 cc->search_order = order;
1608                                 page = freepage;
1609                                 break;
1610                         }
1611
1612                         if (pfn >= min_pfn && pfn > high_pfn) {
1613                                 high_pfn = pfn;
1614
1615                                 /* Shorten the scan if a candidate is found */
1616                                 limit >>= 1;
1617                         }
1618
1619                         if (order_scanned >= limit)
1620                                 break;
1621                 }
1622
1623                 /* Use a maximum candidate pfn if a preferred one was not found */
1624                 if (!page && high_pfn) {
1625                         page = pfn_to_page(high_pfn);
1626
1627                         /* Update freepage for the list reorder below */
1628                         freepage = page;
1629                 }
1630
1631                 /* Reorder to so a future search skips recent pages */
1632                 move_freelist_head(freelist, freepage);
1633
1634                 /* Isolate the page if available */
1635                 if (page) {
1636                         if (__isolate_free_page(page, order)) {
1637                                 set_page_private(page, order);
1638                                 nr_isolated = 1 << order;
1639                                 nr_scanned += nr_isolated - 1;
1640                                 total_isolated += nr_isolated;
1641                                 cc->nr_freepages += nr_isolated;
1642                                 list_add_tail(&page->lru, &cc->freepages[order]);
1643                                 count_compact_events(COMPACTISOLATED, nr_isolated);
1644                         } else {
1645                                 /* If isolation fails, abort the search */
1646                                 order = cc->search_order + 1;
1647                                 page = NULL;
1648                         }
1649                 }
1650
1651                 spin_unlock_irqrestore(&cc->zone->lock, flags);
1652
1653                 /* Skip fast search if enough freepages isolated */
1654                 if (cc->nr_freepages >= cc->nr_migratepages)
1655                         break;
1656
1657                 /*
1658                  * Smaller scan on next order so the total scan is related
1659                  * to freelist_scan_limit.
1660                  */
1661                 if (order_scanned >= limit)
1662                         limit = max(1U, limit >> 1);
1663         }
1664
1665         trace_mm_compaction_fast_isolate_freepages(min_pfn, cc->free_pfn,
1666                                                    nr_scanned, total_isolated);
1667
1668         if (!page) {
1669                 cc->fast_search_fail++;
1670                 if (scan_start) {
1671                         /*
1672                          * Use the highest PFN found above min. If one was
1673                          * not found, be pessimistic for direct compaction
1674                          * and use the min mark.
1675                          */
1676                         if (highest >= min_pfn) {
1677                                 page = pfn_to_page(highest);
1678                                 cc->free_pfn = highest;
1679                         } else {
1680                                 if (cc->direct_compaction && pfn_valid(min_pfn)) {
1681                                         page = pageblock_pfn_to_page(min_pfn,
1682                                                 min(pageblock_end_pfn(min_pfn),
1683                                                     zone_end_pfn(cc->zone)),
1684                                                 cc->zone);
1685                                         if (page && !suitable_migration_target(cc, page))
1686                                                 page = NULL;
1687
1688                                         cc->free_pfn = min_pfn;
1689                                 }
1690                         }
1691                 }
1692         }
1693
1694         if (highest && highest >= cc->zone->compact_cached_free_pfn) {
1695                 highest -= pageblock_nr_pages;
1696                 cc->zone->compact_cached_free_pfn = highest;
1697         }
1698
1699         cc->total_free_scanned += nr_scanned;
1700         if (!page)
1701                 return;
1702
1703         low_pfn = page_to_pfn(page);
1704         fast_isolate_around(cc, low_pfn);
1705 }
1706
1707 /*
1708  * Based on information in the current compact_control, find blocks
1709  * suitable for isolating free pages from and then isolate them.
1710  */
1711 static void isolate_freepages(struct compact_control *cc)
1712 {
1713         struct zone *zone = cc->zone;
1714         struct page *page;
1715         unsigned long block_start_pfn;  /* start of current pageblock */
1716         unsigned long isolate_start_pfn; /* exact pfn we start at */
1717         unsigned long block_end_pfn;    /* end of current pageblock */
1718         unsigned long low_pfn;       /* lowest pfn scanner is able to scan */
1719         unsigned int stride;
1720
1721         /* Try a small search of the free lists for a candidate */
1722         fast_isolate_freepages(cc);
1723         if (cc->nr_freepages)
1724                 return;
1725
1726         /*
1727          * Initialise the free scanner. The starting point is where we last
1728          * successfully isolated from, zone-cached value, or the end of the
1729          * zone when isolating for the first time. For looping we also need
1730          * this pfn aligned down to the pageblock boundary, because we do
1731          * block_start_pfn -= pageblock_nr_pages in the for loop.
1732          * For ending point, take care when isolating in last pageblock of a
1733          * zone which ends in the middle of a pageblock.
1734          * The low boundary is the end of the pageblock the migration scanner
1735          * is using.
1736          */
1737         isolate_start_pfn = cc->free_pfn;
1738         block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
1739         block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1740                                                 zone_end_pfn(zone));
1741         low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1742         stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
1743
1744         /*
1745          * Isolate free pages until enough are available to migrate the
1746          * pages on cc->migratepages. We stop searching if the migrate
1747          * and free page scanners meet or enough free pages are isolated.
1748          */
1749         for (; block_start_pfn >= low_pfn;
1750                                 block_end_pfn = block_start_pfn,
1751                                 block_start_pfn -= pageblock_nr_pages,
1752                                 isolate_start_pfn = block_start_pfn) {
1753                 unsigned long nr_isolated;
1754
1755                 /*
1756                  * This can iterate a massively long zone without finding any
1757                  * suitable migration targets, so periodically check resched.
1758                  */
1759                 if (!(block_start_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
1760                         cond_resched();
1761
1762                 page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1763                                                                         zone);
1764                 if (!page) {
1765                         unsigned long next_pfn;
1766
1767                         next_pfn = skip_offline_sections_reverse(block_start_pfn);
1768                         if (next_pfn)
1769                                 block_start_pfn = max(next_pfn, low_pfn);
1770
1771                         continue;
1772                 }
1773
1774                 /* Check the block is suitable for migration */
1775                 if (!suitable_migration_target(cc, page))
1776                         continue;
1777
1778                 /* If isolation recently failed, do not retry */
1779                 if (!isolation_suitable(cc, page))
1780                         continue;
1781
1782                 /* Found a block suitable for isolating free pages from. */
1783                 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
1784                                         block_end_pfn, cc->freepages, stride, false);
1785
1786                 /* Update the skip hint if the full pageblock was scanned */
1787                 if (isolate_start_pfn == block_end_pfn)
1788                         update_pageblock_skip(cc, page, block_start_pfn -
1789                                               pageblock_nr_pages);
1790
1791                 /* Are enough freepages isolated? */
1792                 if (cc->nr_freepages >= cc->nr_migratepages) {
1793                         if (isolate_start_pfn >= block_end_pfn) {
1794                                 /*
1795                                  * Restart at previous pageblock if more
1796                                  * freepages can be isolated next time.
1797                                  */
1798                                 isolate_start_pfn =
1799                                         block_start_pfn - pageblock_nr_pages;
1800                         }
1801                         break;
1802                 } else if (isolate_start_pfn < block_end_pfn) {
1803                         /*
1804                          * If isolation failed early, do not continue
1805                          * needlessly.
1806                          */
1807                         break;
1808                 }
1809
1810                 /* Adjust stride depending on isolation */
1811                 if (nr_isolated) {
1812                         stride = 1;
1813                         continue;
1814                 }
1815                 stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
1816         }
1817
1818         /*
1819          * Record where the free scanner will restart next time. Either we
1820          * broke from the loop and set isolate_start_pfn based on the last
1821          * call to isolate_freepages_block(), or we met the migration scanner
1822          * and the loop terminated due to isolate_start_pfn < low_pfn
1823          */
1824         cc->free_pfn = isolate_start_pfn;
1825 }
1826
1827 /*
1828  * This is a migrate-callback that "allocates" freepages by taking pages
1829  * from the isolated freelists in the block we are migrating to.
1830  */
1831 static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data)
1832 {
1833         struct compact_control *cc = (struct compact_control *)data;
1834         struct folio *dst;
1835         int order = folio_order(src);
1836         bool has_isolated_pages = false;
1837         int start_order;
1838         struct page *freepage;
1839         unsigned long size;
1840
1841 again:
1842         for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++)
1843                 if (!list_empty(&cc->freepages[start_order]))
1844                         break;
1845
1846         /* no free pages in the list */
1847         if (start_order == NR_PAGE_ORDERS) {
1848                 if (has_isolated_pages)
1849                         return NULL;
1850                 isolate_freepages(cc);
1851                 has_isolated_pages = true;
1852                 goto again;
1853         }
1854
1855         freepage = list_first_entry(&cc->freepages[start_order], struct page,
1856                                 lru);
1857         size = 1 << start_order;
1858
1859         list_del(&freepage->lru);
1860
1861         while (start_order > order) {
1862                 start_order--;
1863                 size >>= 1;
1864
1865                 list_add(&freepage[size].lru, &cc->freepages[start_order]);
1866                 set_page_private(&freepage[size], start_order);
1867         }
1868         dst = (struct folio *)freepage;
1869
1870         post_alloc_hook(&dst->page, order, __GFP_MOVABLE);
1871         if (order)
1872                 prep_compound_page(&dst->page, order);
1873         cc->nr_freepages -= 1 << order;
1874         cc->nr_migratepages -= 1 << order;
1875         return page_rmappable_folio(&dst->page);
1876 }
1877
1878 static struct folio *compaction_alloc(struct folio *src, unsigned long data)
1879 {
1880         return alloc_hooks(compaction_alloc_noprof(src, data));
1881 }
1882
1883 /*
1884  * This is a migrate-callback that "frees" freepages back to the isolated
1885  * freelist.  All pages on the freelist are from the same zone, so there is no
1886  * special handling needed for NUMA.
1887  */
1888 static void compaction_free(struct folio *dst, unsigned long data)
1889 {
1890         struct compact_control *cc = (struct compact_control *)data;
1891         int order = folio_order(dst);
1892         struct page *page = &dst->page;
1893
1894         if (folio_put_testzero(dst)) {
1895                 free_pages_prepare(page, order);
1896                 list_add(&dst->lru, &cc->freepages[order]);
1897                 cc->nr_freepages += 1 << order;
1898         }
1899         cc->nr_migratepages += 1 << order;
1900         /*
1901          * someone else has referenced the page, we cannot take it back to our
1902          * free list.
1903          */
1904 }
1905
1906 /* possible outcome of isolate_migratepages */
1907 typedef enum {
1908         ISOLATE_ABORT,          /* Abort compaction now */
1909         ISOLATE_NONE,           /* No pages isolated, continue scanning */
1910         ISOLATE_SUCCESS,        /* Pages isolated, migrate */
1911 } isolate_migrate_t;
1912
1913 /*
1914  * Allow userspace to control policy on scanning the unevictable LRU for
1915  * compactable pages.
1916  */
1917 static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
1918 /*
1919  * Tunable for proactive compaction. It determines how
1920  * aggressively the kernel should compact memory in the
1921  * background. It takes values in the range [0, 100].
1922  */
1923 static unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
1924 static int sysctl_extfrag_threshold = 500;
1925 static int __read_mostly sysctl_compact_memory;
1926
1927 static inline void
1928 update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
1929 {
1930         if (cc->fast_start_pfn == ULONG_MAX)
1931                 return;
1932
1933         if (!cc->fast_start_pfn)
1934                 cc->fast_start_pfn = pfn;
1935
1936         cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
1937 }
1938
1939 static inline unsigned long
1940 reinit_migrate_pfn(struct compact_control *cc)
1941 {
1942         if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
1943                 return cc->migrate_pfn;
1944
1945         cc->migrate_pfn = cc->fast_start_pfn;
1946         cc->fast_start_pfn = ULONG_MAX;
1947
1948         return cc->migrate_pfn;
1949 }
1950
1951 /*
1952  * Briefly search the free lists for a migration source that already has
1953  * some free pages to reduce the number of pages that need migration
1954  * before a pageblock is free.
1955  */
1956 static unsigned long fast_find_migrateblock(struct compact_control *cc)
1957 {
1958         unsigned int limit = freelist_scan_limit(cc);
1959         unsigned int nr_scanned = 0;
1960         unsigned long distance;
1961         unsigned long pfn = cc->migrate_pfn;
1962         unsigned long high_pfn;
1963         int order;
1964         bool found_block = false;
1965
1966         /* Skip hints are relied on to avoid repeats on the fast search */
1967         if (cc->ignore_skip_hint)
1968                 return pfn;
1969
1970         /*
1971          * If the pageblock should be finished then do not select a different
1972          * pageblock.
1973          */
1974         if (cc->finish_pageblock)
1975                 return pfn;
1976
1977         /*
1978          * If the migrate_pfn is not at the start of a zone or the start
1979          * of a pageblock then assume this is a continuation of a previous
1980          * scan restarted due to COMPACT_CLUSTER_MAX.
1981          */
1982         if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
1983                 return pfn;
1984
1985         /*
1986          * For smaller orders, just linearly scan as the number of pages
1987          * to migrate should be relatively small and does not necessarily
1988          * justify freeing up a large block for a small allocation.
1989          */
1990         if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
1991                 return pfn;
1992
1993         /*
1994          * Only allow kcompactd and direct requests for movable pages to
1995          * quickly clear out a MOVABLE pageblock for allocation. This
1996          * reduces the risk that a large movable pageblock is freed for
1997          * an unmovable/reclaimable small allocation.
1998          */
1999         if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
2000                 return pfn;
2001
2002         /*
2003          * When starting the migration scanner, pick any pageblock within the
2004          * first half of the search space. Otherwise try and pick a pageblock
2005          * within the first eighth to reduce the chances that a migration
2006          * target later becomes a source.
2007          */
2008         distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
2009         if (cc->migrate_pfn != cc->zone->zone_start_pfn)
2010                 distance >>= 2;
2011         high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
2012
2013         for (order = cc->order - 1;
2014              order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;
2015              order--) {
2016                 struct free_area *area = &cc->zone->free_area[order];
2017                 struct list_head *freelist;
2018                 unsigned long flags;
2019                 struct page *freepage;
2020
2021                 if (!area->nr_free)
2022                         continue;
2023
2024                 spin_lock_irqsave(&cc->zone->lock, flags);
2025                 freelist = &area->free_list[MIGRATE_MOVABLE];
2026                 list_for_each_entry(freepage, freelist, buddy_list) {
2027                         unsigned long free_pfn;
2028
2029                         if (nr_scanned++ >= limit) {
2030                                 move_freelist_tail(freelist, freepage);
2031                                 break;
2032                         }
2033
2034                         free_pfn = page_to_pfn(freepage);
2035                         if (free_pfn < high_pfn) {
2036                                 /*
2037                                  * Avoid if skipped recently. Ideally it would
2038                                  * move to the tail but even safe iteration of
2039                                  * the list assumes an entry is deleted, not
2040                                  * reordered.
2041                                  */
2042                                 if (get_pageblock_skip(freepage))
2043                                         continue;
2044
2045                                 /* Reorder to so a future search skips recent pages */
2046                                 move_freelist_tail(freelist, freepage);
2047
2048                                 update_fast_start_pfn(cc, free_pfn);
2049                                 pfn = pageblock_start_pfn(free_pfn);
2050                                 if (pfn < cc->zone->zone_start_pfn)
2051                                         pfn = cc->zone->zone_start_pfn;
2052                                 cc->fast_search_fail = 0;
2053                                 found_block = true;
2054                                 break;
2055                         }
2056                 }
2057                 spin_unlock_irqrestore(&cc->zone->lock, flags);
2058         }
2059
2060         cc->total_migrate_scanned += nr_scanned;
2061
2062         /*
2063          * If fast scanning failed then use a cached entry for a page block
2064          * that had free pages as the basis for starting a linear scan.
2065          */
2066         if (!found_block) {
2067                 cc->fast_search_fail++;
2068                 pfn = reinit_migrate_pfn(cc);
2069         }
2070         return pfn;
2071 }
2072
2073 /*
2074  * Isolate all pages that can be migrated from the first suitable block,
2075  * starting at the block pointed to by the migrate scanner pfn within
2076  * compact_control.
2077  */
2078 static isolate_migrate_t isolate_migratepages(struct compact_control *cc)
2079 {
2080         unsigned long block_start_pfn;
2081         unsigned long block_end_pfn;
2082         unsigned long low_pfn;
2083         struct page *page;
2084         const isolate_mode_t isolate_mode =
2085                 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
2086                 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
2087         bool fast_find_block;
2088
2089         /*
2090          * Start at where we last stopped, or beginning of the zone as
2091          * initialized by compact_zone(). The first failure will use
2092          * the lowest PFN as the starting point for linear scanning.
2093          */
2094         low_pfn = fast_find_migrateblock(cc);
2095         block_start_pfn = pageblock_start_pfn(low_pfn);
2096         if (block_start_pfn < cc->zone->zone_start_pfn)
2097                 block_start_pfn = cc->zone->zone_start_pfn;
2098
2099         /*
2100          * fast_find_migrateblock() has already ensured the pageblock is not
2101          * set with a skipped flag, so to avoid the isolation_suitable check
2102          * below again, check whether the fast search was successful.
2103          */
2104         fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
2105
2106         /* Only scan within a pageblock boundary */
2107         block_end_pfn = pageblock_end_pfn(low_pfn);
2108
2109         /*
2110          * Iterate over whole pageblocks until we find the first suitable.
2111          * Do not cross the free scanner.
2112          */
2113         for (; block_end_pfn <= cc->free_pfn;
2114                         fast_find_block = false,
2115                         cc->migrate_pfn = low_pfn = block_end_pfn,
2116                         block_start_pfn = block_end_pfn,
2117                         block_end_pfn += pageblock_nr_pages) {
2118
2119                 /*
2120                  * This can potentially iterate a massively long zone with
2121                  * many pageblocks unsuitable, so periodically check if we
2122                  * need to schedule.
2123                  */
2124                 if (!(low_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
2125                         cond_resched();
2126
2127                 page = pageblock_pfn_to_page(block_start_pfn,
2128                                                 block_end_pfn, cc->zone);
2129                 if (!page) {
2130                         unsigned long next_pfn;
2131
2132                         next_pfn = skip_offline_sections(block_start_pfn);
2133                         if (next_pfn)
2134                                 block_end_pfn = min(next_pfn, cc->free_pfn);
2135                         continue;
2136                 }
2137
2138                 /*
2139                  * If isolation recently failed, do not retry. Only check the
2140                  * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock
2141                  * to be visited multiple times. Assume skip was checked
2142                  * before making it "skip" so other compaction instances do
2143                  * not scan the same block.
2144                  */
2145                 if ((pageblock_aligned(low_pfn) ||
2146                      low_pfn == cc->zone->zone_start_pfn) &&
2147                     !fast_find_block && !isolation_suitable(cc, page))
2148                         continue;
2149
2150                 /*
2151                  * For async direct compaction, only scan the pageblocks of the
2152                  * same migratetype without huge pages. Async direct compaction
2153                  * is optimistic to see if the minimum amount of work satisfies
2154                  * the allocation. The cached PFN is updated as it's possible
2155                  * that all remaining blocks between source and target are
2156                  * unsuitable and the compaction scanners fail to meet.
2157                  */
2158                 if (!suitable_migration_source(cc, page)) {
2159                         update_cached_migrate(cc, block_end_pfn);
2160                         continue;
2161                 }
2162
2163                 /* Perform the isolation */
2164                 if (isolate_migratepages_block(cc, low_pfn, block_end_pfn,
2165                                                 isolate_mode))
2166                         return ISOLATE_ABORT;
2167
2168                 /*
2169                  * Either we isolated something and proceed with migration. Or
2170                  * we failed and compact_zone should decide if we should
2171                  * continue or not.
2172                  */
2173                 break;
2174         }
2175
2176         return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
2177 }
2178
2179 /*
2180  * Determine whether kswapd is (or recently was!) running on this node.
2181  *
2182  * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't
2183  * zero it.
2184  */
2185 static bool kswapd_is_running(pg_data_t *pgdat)
2186 {
2187         bool running;
2188
2189         pgdat_kswapd_lock(pgdat);
2190         running = pgdat->kswapd && task_is_running(pgdat->kswapd);
2191         pgdat_kswapd_unlock(pgdat);
2192
2193         return running;
2194 }
2195
2196 /*
2197  * A zone's fragmentation score is the external fragmentation wrt to the
2198  * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100].
2199  */
2200 static unsigned int fragmentation_score_zone(struct zone *zone)
2201 {
2202         return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);
2203 }
2204
2205 /*
2206  * A weighted zone's fragmentation score is the external fragmentation
2207  * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It
2208  * returns a value in the range [0, 100].
2209  *
2210  * The scaling factor ensures that proactive compaction focuses on larger
2211  * zones like ZONE_NORMAL, rather than smaller, specialized zones like
2212  * ZONE_DMA32. For smaller zones, the score value remains close to zero,
2213  * and thus never exceeds the high threshold for proactive compaction.
2214  */
2215 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)
2216 {
2217         unsigned long score;
2218
2219         score = zone->present_pages * fragmentation_score_zone(zone);
2220         return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);
2221 }
2222
2223 /*
2224  * The per-node proactive (background) compaction process is started by its
2225  * corresponding kcompactd thread when the node's fragmentation score
2226  * exceeds the high threshold. The compaction process remains active till
2227  * the node's score falls below the low threshold, or one of the back-off
2228  * conditions is met.
2229  */
2230 static unsigned int fragmentation_score_node(pg_data_t *pgdat)
2231 {
2232         unsigned int score = 0;
2233         int zoneid;
2234
2235         for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2236                 struct zone *zone;
2237
2238                 zone = &pgdat->node_zones[zoneid];
2239                 if (!populated_zone(zone))
2240                         continue;
2241                 score += fragmentation_score_zone_weighted(zone);
2242         }
2243
2244         return score;
2245 }
2246
2247 static unsigned int fragmentation_score_wmark(bool low)
2248 {
2249         unsigned int wmark_low;
2250
2251         /*
2252          * Cap the low watermark to avoid excessive compaction
2253          * activity in case a user sets the proactiveness tunable
2254          * close to 100 (maximum).
2255          */
2256         wmark_low = max(100U - sysctl_compaction_proactiveness, 5U);
2257         return low ? wmark_low : min(wmark_low + 10, 100U);
2258 }
2259
2260 static bool should_proactive_compact_node(pg_data_t *pgdat)
2261 {
2262         int wmark_high;
2263
2264         if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))
2265                 return false;
2266
2267         wmark_high = fragmentation_score_wmark(false);
2268         return fragmentation_score_node(pgdat) > wmark_high;
2269 }
2270
2271 static enum compact_result __compact_finished(struct compact_control *cc)
2272 {
2273         unsigned int order;
2274         const int migratetype = cc->migratetype;
2275         int ret;
2276
2277         /* Compaction run completes if the migrate and free scanner meet */
2278         if (compact_scanners_met(cc)) {
2279                 /* Let the next compaction start anew. */
2280                 reset_cached_positions(cc->zone);
2281
2282                 /*
2283                  * Mark that the PG_migrate_skip information should be cleared
2284                  * by kswapd when it goes to sleep. kcompactd does not set the
2285                  * flag itself as the decision to be clear should be directly
2286                  * based on an allocation request.
2287                  */
2288                 if (cc->direct_compaction)
2289                         cc->zone->compact_blockskip_flush = true;
2290
2291                 if (cc->whole_zone)
2292                         return COMPACT_COMPLETE;
2293                 else
2294                         return COMPACT_PARTIAL_SKIPPED;
2295         }
2296
2297         if (cc->proactive_compaction) {
2298                 int score, wmark_low;
2299                 pg_data_t *pgdat;
2300
2301                 pgdat = cc->zone->zone_pgdat;
2302                 if (kswapd_is_running(pgdat))
2303                         return COMPACT_PARTIAL_SKIPPED;
2304
2305                 score = fragmentation_score_zone(cc->zone);
2306                 wmark_low = fragmentation_score_wmark(true);
2307
2308                 if (score > wmark_low)
2309                         ret = COMPACT_CONTINUE;
2310                 else
2311                         ret = COMPACT_SUCCESS;
2312
2313                 goto out;
2314         }
2315
2316         if (is_via_compact_memory(cc->order))
2317                 return COMPACT_CONTINUE;
2318
2319         /*
2320          * Always finish scanning a pageblock to reduce the possibility of
2321          * fallbacks in the future. This is particularly important when
2322          * migration source is unmovable/reclaimable but it's not worth
2323          * special casing.
2324          */
2325         if (!pageblock_aligned(cc->migrate_pfn))
2326                 return COMPACT_CONTINUE;
2327
2328         /* Direct compactor: Is a suitable page free? */
2329         ret = COMPACT_NO_SUITABLE_PAGE;
2330         for (order = cc->order; order < NR_PAGE_ORDERS; order++) {
2331                 struct free_area *area = &cc->zone->free_area[order];
2332                 bool can_steal;
2333
2334                 /* Job done if page is free of the right migratetype */
2335                 if (!free_area_empty(area, migratetype))
2336                         return COMPACT_SUCCESS;
2337
2338 #ifdef CONFIG_CMA
2339                 /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
2340                 if (migratetype == MIGRATE_MOVABLE &&
2341                         !free_area_empty(area, MIGRATE_CMA))
2342                         return COMPACT_SUCCESS;
2343 #endif
2344                 /*
2345                  * Job done if allocation would steal freepages from
2346                  * other migratetype buddy lists.
2347                  */
2348                 if (find_suitable_fallback(area, order, migratetype,
2349                                                 true, &can_steal) != -1)
2350                         /*
2351                          * Movable pages are OK in any pageblock. If we are
2352                          * stealing for a non-movable allocation, make sure
2353                          * we finish compacting the current pageblock first
2354                          * (which is assured by the above migrate_pfn align
2355                          * check) so it is as free as possible and we won't
2356                          * have to steal another one soon.
2357                          */
2358                         return COMPACT_SUCCESS;
2359         }
2360
2361 out:
2362         if (cc->contended || fatal_signal_pending(current))
2363                 ret = COMPACT_CONTENDED;
2364
2365         return ret;
2366 }
2367
2368 static enum compact_result compact_finished(struct compact_control *cc)
2369 {
2370         int ret;
2371
2372         ret = __compact_finished(cc);
2373         trace_mm_compaction_finished(cc->zone, cc->order, ret);
2374         if (ret == COMPACT_NO_SUITABLE_PAGE)
2375                 ret = COMPACT_CONTINUE;
2376
2377         return ret;
2378 }
2379
2380 static bool __compaction_suitable(struct zone *zone, int order,
2381                                   int highest_zoneidx,
2382                                   unsigned long wmark_target)
2383 {
2384         unsigned long watermark;
2385         /*
2386          * Watermarks for order-0 must be met for compaction to be able to
2387          * isolate free pages for migration targets. This means that the
2388          * watermark and alloc_flags have to match, or be more pessimistic than
2389          * the check in __isolate_free_page(). We don't use the direct
2390          * compactor's alloc_flags, as they are not relevant for freepage
2391          * isolation. We however do use the direct compactor's highest_zoneidx
2392          * to skip over zones where lowmem reserves would prevent allocation
2393          * even if compaction succeeds.
2394          * For costly orders, we require low watermark instead of min for
2395          * compaction to proceed to increase its chances.
2396          * ALLOC_CMA is used, as pages in CMA pageblocks are considered
2397          * suitable migration targets
2398          */
2399         watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
2400                                 low_wmark_pages(zone) : min_wmark_pages(zone);
2401         watermark += compact_gap(order);
2402         return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,
2403                                    ALLOC_CMA, wmark_target);
2404 }
2405
2406 /*
2407  * compaction_suitable: Is this suitable to run compaction on this zone now?
2408  */
2409 bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx)
2410 {
2411         enum compact_result compact_result;
2412         bool suitable;
2413
2414         suitable = __compaction_suitable(zone, order, highest_zoneidx,
2415                                          zone_page_state(zone, NR_FREE_PAGES));
2416         /*
2417          * fragmentation index determines if allocation failures are due to
2418          * low memory or external fragmentation
2419          *
2420          * index of -1000 would imply allocations might succeed depending on
2421          * watermarks, but we already failed the high-order watermark check
2422          * index towards 0 implies failure is due to lack of memory
2423          * index towards 1000 implies failure is due to fragmentation
2424          *
2425          * Only compact if a failure would be due to fragmentation. Also
2426          * ignore fragindex for non-costly orders where the alternative to
2427          * a successful reclaim/compaction is OOM. Fragindex and the
2428          * vm.extfrag_threshold sysctl is meant as a heuristic to prevent
2429          * excessive compaction for costly orders, but it should not be at the
2430          * expense of system stability.
2431          */
2432         if (suitable) {
2433                 compact_result = COMPACT_CONTINUE;
2434                 if (order > PAGE_ALLOC_COSTLY_ORDER) {
2435                         int fragindex = fragmentation_index(zone, order);
2436
2437                         if (fragindex >= 0 &&
2438                             fragindex <= sysctl_extfrag_threshold) {
2439                                 suitable = false;
2440                                 compact_result = COMPACT_NOT_SUITABLE_ZONE;
2441                         }
2442                 }
2443         } else {
2444                 compact_result = COMPACT_SKIPPED;
2445         }
2446
2447         trace_mm_compaction_suitable(zone, order, compact_result);
2448
2449         return suitable;
2450 }
2451
2452 bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
2453                 int alloc_flags)
2454 {
2455         struct zone *zone;
2456         struct zoneref *z;
2457
2458         /*
2459          * Make sure at least one zone would pass __compaction_suitable if we continue
2460          * retrying the reclaim.
2461          */
2462         for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
2463                                 ac->highest_zoneidx, ac->nodemask) {
2464                 unsigned long available;
2465
2466                 /*
2467                  * Do not consider all the reclaimable memory because we do not
2468                  * want to trash just for a single high order allocation which
2469                  * is even not guaranteed to appear even if __compaction_suitable
2470                  * is happy about the watermark check.
2471                  */
2472                 available = zone_reclaimable_pages(zone) / order;
2473                 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
2474                 if (__compaction_suitable(zone, order, ac->highest_zoneidx,
2475                                           available))
2476                         return true;
2477         }
2478
2479         return false;
2480 }
2481
2482 /*
2483  * Should we do compaction for target allocation order.
2484  * Return COMPACT_SUCCESS if allocation for target order can be already
2485  * satisfied
2486  * Return COMPACT_SKIPPED if compaction for target order is likely to fail
2487  * Return COMPACT_CONTINUE if compaction for target order should be ran
2488  */
2489 static enum compact_result
2490 compaction_suit_allocation_order(struct zone *zone, unsigned int order,
2491                                  int highest_zoneidx, unsigned int alloc_flags)
2492 {
2493         unsigned long watermark;
2494
2495         watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
2496         if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,
2497                               alloc_flags))
2498                 return COMPACT_SUCCESS;
2499
2500         if (!compaction_suitable(zone, order, highest_zoneidx))
2501                 return COMPACT_SKIPPED;
2502
2503         return COMPACT_CONTINUE;
2504 }
2505
2506 static enum compact_result
2507 compact_zone(struct compact_control *cc, struct capture_control *capc)
2508 {
2509         enum compact_result ret;
2510         unsigned long start_pfn = cc->zone->zone_start_pfn;
2511         unsigned long end_pfn = zone_end_pfn(cc->zone);
2512         unsigned long last_migrated_pfn;
2513         const bool sync = cc->mode != MIGRATE_ASYNC;
2514         bool update_cached;
2515         unsigned int nr_succeeded = 0, nr_migratepages;
2516         int order;
2517
2518         /*
2519          * These counters track activities during zone compaction.  Initialize
2520          * them before compacting a new zone.
2521          */
2522         cc->total_migrate_scanned = 0;
2523         cc->total_free_scanned = 0;
2524         cc->nr_migratepages = 0;
2525         cc->nr_freepages = 0;
2526         for (order = 0; order < NR_PAGE_ORDERS; order++)
2527                 INIT_LIST_HEAD(&cc->freepages[order]);
2528         INIT_LIST_HEAD(&cc->migratepages);
2529
2530         cc->migratetype = gfp_migratetype(cc->gfp_mask);
2531
2532         if (!is_via_compact_memory(cc->order)) {
2533                 ret = compaction_suit_allocation_order(cc->zone, cc->order,
2534                                                        cc->highest_zoneidx,
2535                                                        cc->alloc_flags);
2536                 if (ret != COMPACT_CONTINUE)
2537                         return ret;
2538         }
2539
2540         /*
2541          * Clear pageblock skip if there were failures recently and compaction
2542          * is about to be retried after being deferred.
2543          */
2544         if (compaction_restarting(cc->zone, cc->order))
2545                 __reset_isolation_suitable(cc->zone);
2546
2547         /*
2548          * Setup to move all movable pages to the end of the zone. Used cached
2549          * information on where the scanners should start (unless we explicitly
2550          * want to compact the whole zone), but check that it is initialised
2551          * by ensuring the values are within zone boundaries.
2552          */
2553         cc->fast_start_pfn = 0;
2554         if (cc->whole_zone) {
2555                 cc->migrate_pfn = start_pfn;
2556                 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2557         } else {
2558                 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
2559                 cc->free_pfn = cc->zone->compact_cached_free_pfn;
2560                 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
2561                         cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2562                         cc->zone->compact_cached_free_pfn = cc->free_pfn;
2563                 }
2564                 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
2565                         cc->migrate_pfn = start_pfn;
2566                         cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
2567                         cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
2568                 }
2569
2570                 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
2571                         cc->whole_zone = true;
2572         }
2573
2574         last_migrated_pfn = 0;
2575
2576         /*
2577          * Migrate has separate cached PFNs for ASYNC and SYNC* migration on
2578          * the basis that some migrations will fail in ASYNC mode. However,
2579          * if the cached PFNs match and pageblocks are skipped due to having
2580          * no isolation candidates, then the sync state does not matter.
2581          * Until a pageblock with isolation candidates is found, keep the
2582          * cached PFNs in sync to avoid revisiting the same blocks.
2583          */
2584         update_cached = !sync &&
2585                 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
2586
2587         trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync);
2588
2589         /* lru_add_drain_all could be expensive with involving other CPUs */
2590         lru_add_drain();
2591
2592         while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
2593                 int err;
2594                 unsigned long iteration_start_pfn = cc->migrate_pfn;
2595
2596                 /*
2597                  * Avoid multiple rescans of the same pageblock which can
2598                  * happen if a page cannot be isolated (dirty/writeback in
2599                  * async mode) or if the migrated pages are being allocated
2600                  * before the pageblock is cleared.  The first rescan will
2601                  * capture the entire pageblock for migration. If it fails,
2602                  * it'll be marked skip and scanning will proceed as normal.
2603                  */
2604                 cc->finish_pageblock = false;
2605                 if (pageblock_start_pfn(last_migrated_pfn) ==
2606                     pageblock_start_pfn(iteration_start_pfn)) {
2607                         cc->finish_pageblock = true;
2608                 }
2609
2610 rescan:
2611                 switch (isolate_migratepages(cc)) {
2612                 case ISOLATE_ABORT:
2613                         ret = COMPACT_CONTENDED;
2614                         putback_movable_pages(&cc->migratepages);
2615                         cc->nr_migratepages = 0;
2616                         goto out;
2617                 case ISOLATE_NONE:
2618                         if (update_cached) {
2619                                 cc->zone->compact_cached_migrate_pfn[1] =
2620                                         cc->zone->compact_cached_migrate_pfn[0];
2621                         }
2622
2623                         /*
2624                          * We haven't isolated and migrated anything, but
2625                          * there might still be unflushed migrations from
2626                          * previous cc->order aligned block.
2627                          */
2628                         goto check_drain;
2629                 case ISOLATE_SUCCESS:
2630                         update_cached = false;
2631                         last_migrated_pfn = max(cc->zone->zone_start_pfn,
2632                                 pageblock_start_pfn(cc->migrate_pfn - 1));
2633                 }
2634
2635                 /*
2636                  * Record the number of pages to migrate since the
2637                  * compaction_alloc/free() will update cc->nr_migratepages
2638                  * properly.
2639                  */
2640                 nr_migratepages = cc->nr_migratepages;
2641                 err = migrate_pages(&cc->migratepages, compaction_alloc,
2642                                 compaction_free, (unsigned long)cc, cc->mode,
2643                                 MR_COMPACTION, &nr_succeeded);
2644
2645                 trace_mm_compaction_migratepages(nr_migratepages, nr_succeeded);
2646
2647                 /* All pages were either migrated or will be released */
2648                 cc->nr_migratepages = 0;
2649                 if (err) {
2650                         putback_movable_pages(&cc->migratepages);
2651                         /*
2652                          * migrate_pages() may return -ENOMEM when scanners meet
2653                          * and we want compact_finished() to detect it
2654                          */
2655                         if (err == -ENOMEM && !compact_scanners_met(cc)) {
2656                                 ret = COMPACT_CONTENDED;
2657                                 goto out;
2658                         }
2659                         /*
2660                          * If an ASYNC or SYNC_LIGHT fails to migrate a page
2661                          * within the pageblock_order-aligned block and
2662                          * fast_find_migrateblock may be used then scan the
2663                          * remainder of the pageblock. This will mark the
2664                          * pageblock "skip" to avoid rescanning in the near
2665                          * future. This will isolate more pages than necessary
2666                          * for the request but avoid loops due to
2667                          * fast_find_migrateblock revisiting blocks that were
2668                          * recently partially scanned.
2669                          */
2670                         if (!pageblock_aligned(cc->migrate_pfn) &&
2671                             !cc->ignore_skip_hint && !cc->finish_pageblock &&
2672                             (cc->mode < MIGRATE_SYNC)) {
2673                                 cc->finish_pageblock = true;
2674
2675                                 /*
2676                                  * Draining pcplists does not help THP if
2677                                  * any page failed to migrate. Even after
2678                                  * drain, the pageblock will not be free.
2679                                  */
2680                                 if (cc->order == COMPACTION_HPAGE_ORDER)
2681                                         last_migrated_pfn = 0;
2682
2683                                 goto rescan;
2684                         }
2685                 }
2686
2687                 /* Stop if a page has been captured */
2688                 if (capc && capc->page) {
2689                         ret = COMPACT_SUCCESS;
2690                         break;
2691                 }
2692
2693 check_drain:
2694                 /*
2695                  * Has the migration scanner moved away from the previous
2696                  * cc->order aligned block where we migrated from? If yes,
2697                  * flush the pages that were freed, so that they can merge and
2698                  * compact_finished() can detect immediately if allocation
2699                  * would succeed.
2700                  */
2701                 if (cc->order > 0 && last_migrated_pfn) {
2702                         unsigned long current_block_start =
2703                                 block_start_pfn(cc->migrate_pfn, cc->order);
2704
2705                         if (last_migrated_pfn < current_block_start) {
2706                                 lru_add_drain_cpu_zone(cc->zone);
2707                                 /* No more flushing until we migrate again */
2708                                 last_migrated_pfn = 0;
2709                         }
2710                 }
2711         }
2712
2713 out:
2714         /*
2715          * Release free pages and update where the free scanner should restart,
2716          * so we don't leave any returned pages behind in the next attempt.
2717          */
2718         if (cc->nr_freepages > 0) {
2719                 unsigned long free_pfn = release_free_list(cc->freepages);
2720
2721                 cc->nr_freepages = 0;
2722                 VM_BUG_ON(free_pfn == 0);
2723                 /* The cached pfn is always the first in a pageblock */
2724                 free_pfn = pageblock_start_pfn(free_pfn);
2725                 /*
2726                  * Only go back, not forward. The cached pfn might have been
2727                  * already reset to zone end in compact_finished()
2728                  */
2729                 if (free_pfn > cc->zone->compact_cached_free_pfn)
2730                         cc->zone->compact_cached_free_pfn = free_pfn;
2731         }
2732
2733         count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
2734         count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
2735
2736         trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret);
2737
2738         VM_BUG_ON(!list_empty(&cc->migratepages));
2739
2740         return ret;
2741 }
2742
2743 static enum compact_result compact_zone_order(struct zone *zone, int order,
2744                 gfp_t gfp_mask, enum compact_priority prio,
2745                 unsigned int alloc_flags, int highest_zoneidx,
2746                 struct page **capture)
2747 {
2748         enum compact_result ret;
2749         struct compact_control cc = {
2750                 .order = order,
2751                 .search_order = order,
2752                 .gfp_mask = gfp_mask,
2753                 .zone = zone,
2754                 .mode = (prio == COMPACT_PRIO_ASYNC) ?
2755                                         MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
2756                 .alloc_flags = alloc_flags,
2757                 .highest_zoneidx = highest_zoneidx,
2758                 .direct_compaction = true,
2759                 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
2760                 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
2761                 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
2762         };
2763         struct capture_control capc = {
2764                 .cc = &cc,
2765                 .page = NULL,
2766         };
2767
2768         /*
2769          * Make sure the structs are really initialized before we expose the
2770          * capture control, in case we are interrupted and the interrupt handler
2771          * frees a page.
2772          */
2773         barrier();
2774         WRITE_ONCE(current->capture_control, &capc);
2775
2776         ret = compact_zone(&cc, &capc);
2777
2778         /*
2779          * Make sure we hide capture control first before we read the captured
2780          * page pointer, otherwise an interrupt could free and capture a page
2781          * and we would leak it.
2782          */
2783         WRITE_ONCE(current->capture_control, NULL);
2784         *capture = READ_ONCE(capc.page);
2785         /*
2786          * Technically, it is also possible that compaction is skipped but
2787          * the page is still captured out of luck(IRQ came and freed the page).
2788          * Returning COMPACT_SUCCESS in such cases helps in properly accounting
2789          * the COMPACT[STALL|FAIL] when compaction is skipped.
2790          */
2791         if (*capture)
2792                 ret = COMPACT_SUCCESS;
2793
2794         return ret;
2795 }
2796
2797 /**
2798  * try_to_compact_pages - Direct compact to satisfy a high-order allocation
2799  * @gfp_mask: The GFP mask of the current allocation
2800  * @order: The order of the current allocation
2801  * @alloc_flags: The allocation flags of the current allocation
2802  * @ac: The context of current allocation
2803  * @prio: Determines how hard direct compaction should try to succeed
2804  * @capture: Pointer to free page created by compaction will be stored here
2805  *
2806  * This is the main entry point for direct page compaction.
2807  */
2808 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
2809                 unsigned int alloc_flags, const struct alloc_context *ac,
2810                 enum compact_priority prio, struct page **capture)
2811 {
2812         struct zoneref *z;
2813         struct zone *zone;
2814         enum compact_result rc = COMPACT_SKIPPED;
2815
2816         if (!gfp_compaction_allowed(gfp_mask))
2817                 return COMPACT_SKIPPED;
2818
2819         trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
2820
2821         /* Compact each zone in the list */
2822         for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
2823                                         ac->highest_zoneidx, ac->nodemask) {
2824                 enum compact_result status;
2825
2826                 if (cpusets_enabled() &&
2827                         (alloc_flags & ALLOC_CPUSET) &&
2828                         !__cpuset_zone_allowed(zone, gfp_mask))
2829                                 continue;
2830
2831                 if (prio > MIN_COMPACT_PRIORITY
2832                                         && compaction_deferred(zone, order)) {
2833                         rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
2834                         continue;
2835                 }
2836
2837                 status = compact_zone_order(zone, order, gfp_mask, prio,
2838                                 alloc_flags, ac->highest_zoneidx, capture);
2839                 rc = max(status, rc);
2840
2841                 /* The allocation should succeed, stop compacting */
2842                 if (status == COMPACT_SUCCESS) {
2843                         /*
2844                          * We think the allocation will succeed in this zone,
2845                          * but it is not certain, hence the false. The caller
2846                          * will repeat this with true if allocation indeed
2847                          * succeeds in this zone.
2848                          */
2849                         compaction_defer_reset(zone, order, false);
2850
2851                         break;
2852                 }
2853
2854                 if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
2855                                         status == COMPACT_PARTIAL_SKIPPED))
2856                         /*
2857                          * We think that allocation won't succeed in this zone
2858                          * so we defer compaction there. If it ends up
2859                          * succeeding after all, it will be reset.
2860                          */
2861                         defer_compaction(zone, order);
2862
2863                 /*
2864                  * We might have stopped compacting due to need_resched() in
2865                  * async compaction, or due to a fatal signal detected. In that
2866                  * case do not try further zones
2867                  */
2868                 if ((prio == COMPACT_PRIO_ASYNC && need_resched())
2869                                         || fatal_signal_pending(current))
2870                         break;
2871         }
2872
2873         return rc;
2874 }
2875
2876 /*
2877  * compact_node() - compact all zones within a node
2878  * @pgdat: The node page data
2879  * @proactive: Whether the compaction is proactive
2880  *
2881  * For proactive compaction, compact till each zone's fragmentation score
2882  * reaches within proactive compaction thresholds (as determined by the
2883  * proactiveness tunable), it is possible that the function returns before
2884  * reaching score targets due to various back-off conditions, such as,
2885  * contention on per-node or per-zone locks.
2886  */
2887 static int compact_node(pg_data_t *pgdat, bool proactive)
2888 {
2889         int zoneid;
2890         struct zone *zone;
2891         struct compact_control cc = {
2892                 .order = -1,
2893                 .mode = proactive ? MIGRATE_SYNC_LIGHT : MIGRATE_SYNC,
2894                 .ignore_skip_hint = true,
2895                 .whole_zone = true,
2896                 .gfp_mask = GFP_KERNEL,
2897                 .proactive_compaction = proactive,
2898         };
2899
2900         for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
2901                 zone = &pgdat->node_zones[zoneid];
2902                 if (!populated_zone(zone))
2903                         continue;
2904
2905                 if (fatal_signal_pending(current))
2906                         return -EINTR;
2907
2908                 cc.zone = zone;
2909
2910                 compact_zone(&cc, NULL);
2911
2912                 if (proactive) {
2913                         count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
2914                                              cc.total_migrate_scanned);
2915                         count_compact_events(KCOMPACTD_FREE_SCANNED,
2916                                              cc.total_free_scanned);
2917                 }
2918         }
2919
2920         return 0;
2921 }
2922
2923 /* Compact all zones of all nodes in the system */
2924 static int compact_nodes(void)
2925 {
2926         int ret, nid;
2927
2928         /* Flush pending updates to the LRU lists */
2929         lru_add_drain_all();
2930
2931         for_each_online_node(nid) {
2932                 ret = compact_node(NODE_DATA(nid), false);
2933                 if (ret)
2934                         return ret;
2935         }
2936
2937         return 0;
2938 }
2939
2940 static int compaction_proactiveness_sysctl_handler(const struct ctl_table *table, int write,
2941                 void *buffer, size_t *length, loff_t *ppos)
2942 {
2943         int rc, nid;
2944
2945         rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
2946         if (rc)
2947                 return rc;
2948
2949         if (write && sysctl_compaction_proactiveness) {
2950                 for_each_online_node(nid) {
2951                         pg_data_t *pgdat = NODE_DATA(nid);
2952
2953                         if (pgdat->proactive_compact_trigger)
2954                                 continue;
2955
2956                         pgdat->proactive_compact_trigger = true;
2957                         trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, -1,
2958                                                              pgdat->nr_zones - 1);
2959                         wake_up_interruptible(&pgdat->kcompactd_wait);
2960                 }
2961         }
2962
2963         return 0;
2964 }
2965
2966 /*
2967  * This is the entry point for compacting all nodes via
2968  * /proc/sys/vm/compact_memory
2969  */
2970 static int sysctl_compaction_handler(const struct ctl_table *table, int write,
2971                         void *buffer, size_t *length, loff_t *ppos)
2972 {
2973         int ret;
2974
2975         ret = proc_dointvec(table, write, buffer, length, ppos);
2976         if (ret)
2977                 return ret;
2978
2979         if (sysctl_compact_memory != 1)
2980                 return -EINVAL;
2981
2982         if (write)
2983                 ret = compact_nodes();
2984
2985         return ret;
2986 }
2987
2988 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
2989 static ssize_t compact_store(struct device *dev,
2990                              struct device_attribute *attr,
2991                              const char *buf, size_t count)
2992 {
2993         int nid = dev->id;
2994
2995         if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
2996                 /* Flush pending updates to the LRU lists */
2997                 lru_add_drain_all();
2998
2999                 compact_node(NODE_DATA(nid), false);
3000         }
3001
3002         return count;
3003 }
3004 static DEVICE_ATTR_WO(compact);
3005
3006 int compaction_register_node(struct node *node)
3007 {
3008         return device_create_file(&node->dev, &dev_attr_compact);
3009 }
3010
3011 void compaction_unregister_node(struct node *node)
3012 {
3013         device_remove_file(&node->dev, &dev_attr_compact);
3014 }
3015 #endif /* CONFIG_SYSFS && CONFIG_NUMA */
3016
3017 static inline bool kcompactd_work_requested(pg_data_t *pgdat)
3018 {
3019         return pgdat->kcompactd_max_order > 0 || kthread_should_stop() ||
3020                 pgdat->proactive_compact_trigger;
3021 }
3022
3023 static bool kcompactd_node_suitable(pg_data_t *pgdat)
3024 {
3025         int zoneid;
3026         struct zone *zone;
3027         enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;
3028         enum compact_result ret;
3029
3030         for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {
3031                 zone = &pgdat->node_zones[zoneid];
3032
3033                 if (!populated_zone(zone))
3034                         continue;
3035
3036                 ret = compaction_suit_allocation_order(zone,
3037                                 pgdat->kcompactd_max_order,
3038                                 highest_zoneidx, ALLOC_WMARK_MIN);
3039                 if (ret == COMPACT_CONTINUE)
3040                         return true;
3041         }
3042
3043         return false;
3044 }
3045
3046 static void kcompactd_do_work(pg_data_t *pgdat)
3047 {
3048         /*
3049          * With no special task, compact all zones so that a page of requested
3050          * order is allocatable.
3051          */
3052         int zoneid;
3053         struct zone *zone;
3054         struct compact_control cc = {
3055                 .order = pgdat->kcompactd_max_order,
3056                 .search_order = pgdat->kcompactd_max_order,
3057                 .highest_zoneidx = pgdat->kcompactd_highest_zoneidx,
3058                 .mode = MIGRATE_SYNC_LIGHT,
3059                 .ignore_skip_hint = false,
3060                 .gfp_mask = GFP_KERNEL,
3061         };
3062         enum compact_result ret;
3063
3064         trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
3065                                                         cc.highest_zoneidx);
3066         count_compact_event(KCOMPACTD_WAKE);
3067
3068         for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) {
3069                 int status;
3070
3071                 zone = &pgdat->node_zones[zoneid];
3072                 if (!populated_zone(zone))
3073                         continue;
3074
3075                 if (compaction_deferred(zone, cc.order))
3076                         continue;
3077
3078                 ret = compaction_suit_allocation_order(zone,
3079                                 cc.order, zoneid, ALLOC_WMARK_MIN);
3080                 if (ret != COMPACT_CONTINUE)
3081                         continue;
3082
3083                 if (kthread_should_stop())
3084                         return;
3085
3086                 cc.zone = zone;
3087                 status = compact_zone(&cc, NULL);
3088
3089                 if (status == COMPACT_SUCCESS) {
3090                         compaction_defer_reset(zone, cc.order, false);
3091                 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
3092                         /*
3093                          * Buddy pages may become stranded on pcps that could
3094                          * otherwise coalesce on the zone's free area for
3095                          * order >= cc.order.  This is ratelimited by the
3096                          * upcoming deferral.
3097                          */
3098                         drain_all_pages(zone);
3099
3100                         /*
3101                          * We use sync migration mode here, so we defer like
3102                          * sync direct compaction does.
3103                          */
3104                         defer_compaction(zone, cc.order);
3105                 }
3106
3107                 count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
3108                                      cc.total_migrate_scanned);
3109                 count_compact_events(KCOMPACTD_FREE_SCANNED,
3110                                      cc.total_free_scanned);
3111         }
3112
3113         /*
3114          * Regardless of success, we are done until woken up next. But remember
3115          * the requested order/highest_zoneidx in case it was higher/tighter
3116          * than our current ones
3117          */
3118         if (pgdat->kcompactd_max_order <= cc.order)
3119                 pgdat->kcompactd_max_order = 0;
3120         if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)
3121                 pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
3122 }
3123
3124 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)
3125 {
3126         if (!order)
3127                 return;
3128
3129         if (pgdat->kcompactd_max_order < order)
3130                 pgdat->kcompactd_max_order = order;
3131
3132         if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)
3133                 pgdat->kcompactd_highest_zoneidx = highest_zoneidx;
3134
3135         /*
3136          * Pairs with implicit barrier in wait_event_freezable()
3137          * such that wakeups are not missed.
3138          */
3139         if (!wq_has_sleeper(&pgdat->kcompactd_wait))
3140                 return;
3141
3142         if (!kcompactd_node_suitable(pgdat))
3143                 return;
3144
3145         trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
3146                                                         highest_zoneidx);
3147         wake_up_interruptible(&pgdat->kcompactd_wait);
3148 }
3149
3150 /*
3151  * The background compaction daemon, started as a kernel thread
3152  * from the init process.
3153  */
3154 static int kcompactd(void *p)
3155 {
3156         pg_data_t *pgdat = (pg_data_t *)p;
3157         struct task_struct *tsk = current;
3158         long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC);
3159         long timeout = default_timeout;
3160
3161         const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3162
3163         if (!cpumask_empty(cpumask))
3164                 set_cpus_allowed_ptr(tsk, cpumask);
3165
3166         set_freezable();
3167
3168         pgdat->kcompactd_max_order = 0;
3169         pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
3170
3171         while (!kthread_should_stop()) {
3172                 unsigned long pflags;
3173
3174                 /*
3175                  * Avoid the unnecessary wakeup for proactive compaction
3176                  * when it is disabled.
3177                  */
3178                 if (!sysctl_compaction_proactiveness)
3179                         timeout = MAX_SCHEDULE_TIMEOUT;
3180                 trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
3181                 if (wait_event_freezable_timeout(pgdat->kcompactd_wait,
3182                         kcompactd_work_requested(pgdat), timeout) &&
3183                         !pgdat->proactive_compact_trigger) {
3184
3185                         psi_memstall_enter(&pflags);
3186                         kcompactd_do_work(pgdat);
3187                         psi_memstall_leave(&pflags);
3188                         /*
3189                          * Reset the timeout value. The defer timeout from
3190                          * proactive compaction is lost here but that is fine
3191                          * as the condition of the zone changing substantionally
3192                          * then carrying on with the previous defer interval is
3193                          * not useful.
3194                          */
3195                         timeout = default_timeout;
3196                         continue;
3197                 }
3198
3199                 /*
3200                  * Start the proactive work with default timeout. Based
3201                  * on the fragmentation score, this timeout is updated.
3202                  */
3203                 timeout = default_timeout;
3204                 if (should_proactive_compact_node(pgdat)) {
3205                         unsigned int prev_score, score;
3206
3207                         prev_score = fragmentation_score_node(pgdat);
3208                         compact_node(pgdat, true);
3209                         score = fragmentation_score_node(pgdat);
3210                         /*
3211                          * Defer proactive compaction if the fragmentation
3212                          * score did not go down i.e. no progress made.
3213                          */
3214                         if (unlikely(score >= prev_score))
3215                                 timeout =
3216                                    default_timeout << COMPACT_MAX_DEFER_SHIFT;
3217                 }
3218                 if (unlikely(pgdat->proactive_compact_trigger))
3219                         pgdat->proactive_compact_trigger = false;
3220         }
3221
3222         return 0;
3223 }
3224
3225 /*
3226  * This kcompactd start function will be called by init and node-hot-add.
3227  * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
3228  */
3229 void __meminit kcompactd_run(int nid)
3230 {
3231         pg_data_t *pgdat = NODE_DATA(nid);
3232
3233         if (pgdat->kcompactd)
3234                 return;
3235
3236         pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
3237         if (IS_ERR(pgdat->kcompactd)) {
3238                 pr_err("Failed to start kcompactd on node %d\n", nid);
3239                 pgdat->kcompactd = NULL;
3240         }
3241 }
3242
3243 /*
3244  * Called by memory hotplug when all memory in a node is offlined. Caller must
3245  * be holding mem_hotplug_begin/done().
3246  */
3247 void __meminit kcompactd_stop(int nid)
3248 {
3249         struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
3250
3251         if (kcompactd) {
3252                 kthread_stop(kcompactd);
3253                 NODE_DATA(nid)->kcompactd = NULL;
3254         }
3255 }
3256
3257 /*
3258  * It's optimal to keep kcompactd on the same CPUs as their memory, but
3259  * not required for correctness. So if the last cpu in a node goes
3260  * away, we get changed to run anywhere: as the first one comes back,
3261  * restore their cpu bindings.
3262  */
3263 static int kcompactd_cpu_online(unsigned int cpu)
3264 {
3265         int nid;
3266
3267         for_each_node_state(nid, N_MEMORY) {
3268                 pg_data_t *pgdat = NODE_DATA(nid);
3269                 const struct cpumask *mask;
3270
3271                 mask = cpumask_of_node(pgdat->node_id);
3272
3273                 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3274                         /* One of our CPUs online: restore mask */
3275                         if (pgdat->kcompactd)
3276                                 set_cpus_allowed_ptr(pgdat->kcompactd, mask);
3277         }
3278         return 0;
3279 }
3280
3281 static int proc_dointvec_minmax_warn_RT_change(const struct ctl_table *table,
3282                 int write, void *buffer, size_t *lenp, loff_t *ppos)
3283 {
3284         int ret, old;
3285
3286         if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write)
3287                 return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3288
3289         old = *(int *)table->data;
3290         ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3291         if (ret)
3292                 return ret;
3293         if (old != *(int *)table->data)
3294                 pr_warn_once("sysctl attribute %s changed by %s[%d]\n",
3295                              table->procname, current->comm,
3296                              task_pid_nr(current));
3297         return ret;
3298 }
3299
3300 static struct ctl_table vm_compaction[] = {
3301         {
3302                 .procname       = "compact_memory",
3303                 .data           = &sysctl_compact_memory,
3304                 .maxlen         = sizeof(int),
3305                 .mode           = 0200,
3306                 .proc_handler   = sysctl_compaction_handler,
3307         },
3308         {
3309                 .procname       = "compaction_proactiveness",
3310                 .data           = &sysctl_compaction_proactiveness,
3311                 .maxlen         = sizeof(sysctl_compaction_proactiveness),
3312                 .mode           = 0644,
3313                 .proc_handler   = compaction_proactiveness_sysctl_handler,
3314                 .extra1         = SYSCTL_ZERO,
3315                 .extra2         = SYSCTL_ONE_HUNDRED,
3316         },
3317         {
3318                 .procname       = "extfrag_threshold",
3319                 .data           = &sysctl_extfrag_threshold,
3320                 .maxlen         = sizeof(int),
3321                 .mode           = 0644,
3322                 .proc_handler   = proc_dointvec_minmax,
3323                 .extra1         = SYSCTL_ZERO,
3324                 .extra2         = SYSCTL_ONE_THOUSAND,
3325         },
3326         {
3327                 .procname       = "compact_unevictable_allowed",
3328                 .data           = &sysctl_compact_unevictable_allowed,
3329                 .maxlen         = sizeof(int),
3330                 .mode           = 0644,
3331                 .proc_handler   = proc_dointvec_minmax_warn_RT_change,
3332                 .extra1         = SYSCTL_ZERO,
3333                 .extra2         = SYSCTL_ONE,
3334         },
3335 };
3336
3337 static int __init kcompactd_init(void)
3338 {
3339         int nid;
3340         int ret;
3341
3342         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
3343                                         "mm/compaction:online",
3344                                         kcompactd_cpu_online, NULL);
3345         if (ret < 0) {
3346                 pr_err("kcompactd: failed to register hotplug callbacks.\n");
3347                 return ret;
3348         }
3349
3350         for_each_node_state(nid, N_MEMORY)
3351                 kcompactd_run(nid);
3352         register_sysctl_init("vm", vm_compaction);
3353         return 0;
3354 }
3355 subsys_initcall(kcompactd_init)
3356
3357 #endif /* CONFIG_COMPACTION */
This page took 0.223997 seconds and 4 git commands to generate.