mm/bootmem_info.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Bootmem core functions.
   4  *
   5  * Copyright (c) 2020, Bytedance.
   6  *
   7  *     Author: Muchun Song <[email protected]>
   8  *
   9  */
  10 #include <linux/mm.h>
  11 #include <linux/compiler.h>
  12 #include <linux/memblock.h>
  13 #include <linux/bootmem_info.h>
  14 #include <linux/memory_hotplug.h>
  15 #include <linux/kmemleak.h>
  16
  17 void get_page_bootmem(unsigned long info, struct page *page,
  18                 enum bootmem_type type)
  19 {
  20         BUG_ON(type > 0xf);
  21         BUG_ON(info > (ULONG_MAX >> 4));
  22         SetPagePrivate(page);
  23         set_page_private(page, info << 4 | type);
  24         page_ref_inc(page);
  25 }
  26
  27 void put_page_bootmem(struct page *page)
  28 {
  29         enum bootmem_type type = bootmem_type(page);
  30
  31         BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
  32                type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
  33
  34         if (page_ref_dec_return(page) == 1) {
  35                 ClearPagePrivate(page);
  36                 set_page_private(page, 0);
  37                 INIT_LIST_HEAD(&page->lru);
  38                 kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
  39                 free_reserved_page(page);
  40         }
  41 }
  42
  43 #ifndef CONFIG_SPARSEMEM_VMEMMAP
  44 static void __init register_page_bootmem_info_section(unsigned long start_pfn)
  45 {
  46         unsigned long mapsize, section_nr, i;
  47         struct mem_section *ms;
  48         struct page *page, *memmap;
  49         struct mem_section_usage *usage;
  50
  51         section_nr = pfn_to_section_nr(start_pfn);
  52         ms = __nr_to_section(section_nr);
  53
  54         /* Get section's memmap address */
  55         memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  56
  57         /*
  58          * Get page for the memmap's phys address
  59          * XXX: need more consideration for sparse_vmemmap...
  60          */
  61         page = virt_to_page(memmap);
  62         mapsize = sizeof(struct page) * PAGES_PER_SECTION;
  63         mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
  64
  65         /* remember memmap's page */
  66         for (i = 0; i < mapsize; i++, page++)
  67                 get_page_bootmem(section_nr, page, SECTION_INFO);
  68
  69         usage = ms->usage;
  70         page = virt_to_page(usage);
  71
  72         mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
  73
  74         for (i = 0; i < mapsize; i++, page++)
  75                 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  76
  77 }
  78 #else /* CONFIG_SPARSEMEM_VMEMMAP */
  79 static void __init register_page_bootmem_info_section(unsigned long start_pfn)
  80 {
  81         unsigned long mapsize, section_nr, i;
  82         struct mem_section *ms;
  83         struct page *page, *memmap;
  84         struct mem_section_usage *usage;
  85
  86         section_nr = pfn_to_section_nr(start_pfn);
  87         ms = __nr_to_section(section_nr);
  88
  89         memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  90
  91         register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
  92
  93         usage = ms->usage;
  94         page = virt_to_page(usage);
  95
  96         mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
  97
  98         for (i = 0; i < mapsize; i++, page++)
  99                 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
 100 }
 101 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
 102
 103 void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
 104 {
 105         unsigned long i, pfn, end_pfn, nr_pages;
 106         int node = pgdat->node_id;
 107         struct page *page;
 108
 109         nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
 110         page = virt_to_page(pgdat);
 111
 112         for (i = 0; i < nr_pages; i++, page++)
 113                 get_page_bootmem(node, page, NODE_INFO);
 114
 115         pfn = pgdat->node_start_pfn;
 116         end_pfn = pgdat_end_pfn(pgdat);
 117
 118         /* register section info */
 119         for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
 120                 /*
 121                  * Some platforms can assign the same pfn to multiple nodes - on
 122                  * node0 as well as nodeN.  To avoid registering a pfn against
 123                  * multiple nodes we check that this pfn does not already
 124                  * reside in some other nodes.
 125                  */
 126                 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
 127                         register_page_bootmem_info_section(pfn);
 128         }
 129 }