[linux.git] / mm / bootmem_info.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Bootmem core functions.
 *
 * Copyright (c) 2020, Bytedance.
 *
 *     Author: Muchun Song <[email protected]>
 *
 */
#include <linux/mm.h>
#include <linux/compiler.h>
#include <linux/memblock.h>
#include <linux/bootmem_info.h>
#include <linux/memory_hotplug.h>
#include <linux/kmemleak.h>

void get_page_bootmem(unsigned long info, struct page *page, unsigned long type)
{
	page->index = type;
	SetPagePrivate(page);
	set_page_private(page, info);
	page_ref_inc(page);
}

void put_page_bootmem(struct page *page)
{
	unsigned long type = page->index;

	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);

	if (page_ref_dec_return(page) == 1) {
		page->index = 0;
		ClearPagePrivate(page);
		set_page_private(page, 0);
		INIT_LIST_HEAD(&page->lru);
		kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
		free_reserved_page(page);
	}
}

#ifndef CONFIG_SPARSEMEM_VMEMMAP
static void __init register_page_bootmem_info_section(unsigned long start_pfn)
{
	unsigned long mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;
	struct mem_section_usage *usage;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	/* Get section's memmap address */
	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	/*
	 * Get page for the memmap's phys address
	 * XXX: need more consideration for sparse_vmemmap...
	 */
	page = virt_to_page(memmap);
	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;

	/* remember memmap's page */
	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, SECTION_INFO);

	usage = ms->usage;
	page = virt_to_page(usage);

	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);

}
#else /* CONFIG_SPARSEMEM_VMEMMAP */
static void __init register_page_bootmem_info_section(unsigned long start_pfn)
{
	unsigned long mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;
	struct mem_section_usage *usage;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);

	usage = ms->usage;
	page = virt_to_page(usage);

	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
}
#endif /* !CONFIG_SPARSEMEM_VMEMMAP */

void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
{
	unsigned long i, pfn, end_pfn, nr_pages;
	int node = pgdat->node_id;
	struct page *page;

	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
	page = virt_to_page(pgdat);

	for (i = 0; i < nr_pages; i++, page++)
		get_page_bootmem(node, page, NODE_INFO);

	pfn = pgdat->node_start_pfn;
	end_pfn = pgdat_end_pfn(pgdat);

	/* register section info */
	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		/*
		 * Some platforms can assign the same pfn to multiple nodes - on
		 * node0 as well as nodeN.  To avoid registering a pfn against
		 * multiple nodes we check that this pfn does not already
		 * reside in some other nodes.
		 */
		if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
			register_page_bootmem_info_section(pfn);
	}
}
Commit	Line	Data
426e5c42 MS	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Bootmem core functions.
	4	*
	5	* Copyright (c) 2020, Bytedance.
	6	*
	7	* Author: Muchun Song <[email protected]>
	8	*
	9	*/
	10	#include <linux/mm.h>
	11	#include <linux/compiler.h>
	12	#include <linux/memblock.h>
	13	#include <linux/bootmem_info.h>
	14	#include <linux/memory_hotplug.h>
dd0ff4d1	15	#include <linux/kmemleak.h>
426e5c42 MS	16
	17	void get_page_bootmem(unsigned long info, struct page *page, unsigned long type)
	18	{
c5e97ed1	19	page->index = type;
426e5c42 MS	20	SetPagePrivate(page);
	21	set_page_private(page, info);
	22	page_ref_inc(page);
	23	}
	24
	25	void put_page_bootmem(struct page *page)
	26	{
c5e97ed1	27	unsigned long type = page->index;
426e5c42	28
426e5c42 MS	29	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE \|\|
	30	type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
	31
	32	if (page_ref_dec_return(page) == 1) {
c5e97ed1	33	page->index = 0;
426e5c42 MS	34	ClearPagePrivate(page);
	35	set_page_private(page, 0);
	36	INIT_LIST_HEAD(&page->lru);
6d4e2cda	37	kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
426e5c42 MS	38	free_reserved_page(page);
	39	}
	40	}
	41
	42	#ifndef CONFIG_SPARSEMEM_VMEMMAP
cdcfc631	43	static void __init register_page_bootmem_info_section(unsigned long start_pfn)
426e5c42 MS	44	{
	45	unsigned long mapsize, section_nr, i;
	46	struct mem_section *ms;
	47	struct page page, memmap;
	48	struct mem_section_usage *usage;
	49
	50	section_nr = pfn_to_section_nr(start_pfn);
	51	ms = __nr_to_section(section_nr);
	52
	53	/* Get section's memmap address */
	54	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
	55
	56	/*
	57	* Get page for the memmap's phys address
	58	* XXX: need more consideration for sparse_vmemmap...
	59	*/
	60	page = virt_to_page(memmap);
	61	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
	62	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
	63
	64	/* remember memmap's page */
	65	for (i = 0; i < mapsize; i++, page++)
	66	get_page_bootmem(section_nr, page, SECTION_INFO);
	67
	68	usage = ms->usage;
	69	page = virt_to_page(usage);
	70
	71	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
	72
	73	for (i = 0; i < mapsize; i++, page++)
	74	get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
	75
	76	}
	77	#else /* CONFIG_SPARSEMEM_VMEMMAP */
cdcfc631	78	static void __init register_page_bootmem_info_section(unsigned long start_pfn)
426e5c42 MS	79	{
	80	unsigned long mapsize, section_nr, i;
	81	struct mem_section *ms;
	82	struct page page, memmap;
	83	struct mem_section_usage *usage;
	84
	85	section_nr = pfn_to_section_nr(start_pfn);
	86	ms = __nr_to_section(section_nr);
	87
	88	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
	89
	90	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
	91
	92	usage = ms->usage;
	93	page = virt_to_page(usage);
	94
	95	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
	96
	97	for (i = 0; i < mapsize; i++, page++)
	98	get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
	99	}
	100	#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
	101
	102	void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
	103	{
	104	unsigned long i, pfn, end_pfn, nr_pages;
	105	int node = pgdat->node_id;
	106	struct page *page;
	107
	108	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
	109	page = virt_to_page(pgdat);
	110
	111	for (i = 0; i < nr_pages; i++, page++)
	112	get_page_bootmem(node, page, NODE_INFO);
	113
	114	pfn = pgdat->node_start_pfn;
	115	end_pfn = pgdat_end_pfn(pgdat);
	116
	117	/* register section info */
	118	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
	119	/*
	120	* Some platforms can assign the same pfn to multiple nodes - on
	121	* node0 as well as nodeN. To avoid registering a pfn against
	122	* multiple nodes we check that this pfn does not already
	123	* reside in some other nodes.
	124	*/
	125	if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
	126	register_page_bootmem_info_section(pfn);
	127	}
	128	}