mm: free up a word in the first tail page

Store the folio order in the low byte of the flags word in the first tail
page.  This frees up the word that was being used to store the order and
dtor bytes previously.

Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Matthew Wilcox (Oracle) <[email protected]>
Cc: David Hildenbrand <[email protected]>
Cc: Jens Axboe <[email protected]>
Cc: Sidhartha Kumar <[email protected]>
Cc: Yanteng Si <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 63d5737819735d600a9d28bbcd5e7c5891e5614d..939386e0aedab9f018d8fae4fe8ead037d08f7ec 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1000,7 +1000,7 @@ struct inode;
  * compound_order() can be called without holding a reference, which means
  * that niceties like page_folio() don't work.  These callers should be
  * prepared to handle wild return values.  For example, PG_head may be
- * set before _folio_order is initialised, or this may be a tail page.
+ * set before the order is initialised, or this may be a tail page.
  * See compaction.c for some good examples.
  */
 static inline unsigned int compound_order(struct page *page)
@@ -1009,7 +1009,7 @@ static inline unsigned int compound_order(struct page *page)
 
        if (!test_bit(PG_head, &folio->flags))
                return 0;
-       return folio->_folio_order;
+       return folio->_flags_1 & 0xff;
 }
 
 /**
@@ -1025,7 +1025,7 @@ static inline unsigned int folio_order(struct folio *folio)
 {
        if (!folio_test_large(folio))
                return 0;
-       return folio->_folio_order;
+       return folio->_flags_1 & 0xff;
 }
 
 #include <linux/huge_mm.h>
@@ -1996,7 +1996,7 @@ static inline long folio_nr_pages(struct folio *folio)
 #ifdef CONFIG_64BIT
        return folio->_folio_nr_pages;
 #else
-       return 1L << folio->_folio_order;
+       return 1L << (folio->_flags_1 & 0xff);
 #endif
 }
 
@@ -2014,7 +2014,7 @@ static inline unsigned long compound_nr(struct page *page)
 #ifdef CONFIG_64BIT
        return folio->_folio_nr_pages;
 #else
-       return 1L << folio->_folio_order;
+       return 1L << (folio->_flags_1 & 0xff);
 #endif
 }
 

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 1c5c2349c18ea19de02b3c89afcce2d17a4e5505..40afb3bbc309db40a8b541cdcb70cb134607312a 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -264,7 +264,6 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page)
  * @_refcount: Do not access this member directly.  Use folio_ref_count()
  *    to find how many references there are to this folio.
  * @memcg_data: Memory Control Group data.
- * @_folio_order: Do not use directly, call folio_order().
  * @_entire_mapcount: Do not use directly, call folio_entire_mapcount().
  * @_nr_pages_mapped: Do not use directly, call folio_mapcount().
  * @_pincount: Do not use directly, call folio_maybe_dma_pinned().
@@ -316,8 +315,8 @@ struct folio {
                struct {
                        unsigned long _flags_1;
                        unsigned long _head_1;
+                       unsigned long _folio_avail;
        /* public: */
-                       unsigned char _folio_order;
                        atomic_t _entire_mapcount;
                        atomic_t _nr_pages_mapped;
                        atomic_t _pincount;

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 46fc05c648ff517dedb7581c5575cb7828d4b8bd..638b0a96b4c59034480092a24f5b5b6fb3f5920d 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -184,7 +184,8 @@ enum pageflags {
 
        /*
         * Flags only valid for compound pages.  Stored in first tail page's
-        * flags word.
+        * flags word.  Cannot use the first 8 flags or any flag marked as
+        * PF_ANY.
         */
 
        /* At least one page in this folio has the hwpoison flag set */
@@ -1081,8 +1082,8 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
  * the CHECK_AT_FREE flags above, so need to be cleared.
  */
 #define PAGE_FLAGS_SECOND                                              \
-       (1UL << PG_has_hwpoisoned       | 1UL << PG_hugetlb |           \
-        1UL << PG_large_rmappable)
+       (0xffUL /* order */             | 1UL << PG_has_hwpoisoned |    \
+        1UL << PG_hugetlb              | 1UL << PG_large_rmappable)
 
 #define PAGE_FLAGS_PRIVATE                             \
        (1UL << PG_private | 1UL << PG_private_2)

diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index 934dd86e19f594ab830f359cb88377819f8c71d3..693445e1f7f6fd13e1d91ffccd1f02d89ea12b53 100644 (file)
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -455,7 +455,6 @@ static int __init crash_save_vmcoreinfo_init(void)
        VMCOREINFO_OFFSET(page, lru);
        VMCOREINFO_OFFSET(page, _mapcount);
        VMCOREINFO_OFFSET(page, private);
-       VMCOREINFO_OFFSET(folio, _folio_order);
        VMCOREINFO_OFFSET(page, compound_head);
        VMCOREINFO_OFFSET(pglist_data, node_zones);
        VMCOREINFO_OFFSET(pglist_data, nr_zones);

diff --git a/mm/internal.h b/mm/internal.h
index 5c0daea731f34b64cb216c26fcd1a23234068662..d1d4bf4e63c088a6ce3ff637a3778e4cd72a360c 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -407,7 +407,7 @@ static inline void folio_set_order(struct folio *folio, unsigned int order)
        if (WARN_ON_ONCE(!order || !folio_test_large(folio)))
                return;
 
-       folio->_folio_order = order;
+       folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order;
 #ifdef CONFIG_64BIT
        folio->_folio_nr_pages = 1U << order;
 #endif