[linux.git] / arch / s390 / mm / kasan_init.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kasan.h>
#include <linux/sched/task.h>
#include <linux/memblock.h>
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/kasan.h>
#include <asm/mem_detect.h>
#include <asm/processor.h>
#include <asm/sclp.h>
#include <asm/facility.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/uv.h>

unsigned long kasan_vmax;
static unsigned long segment_pos __initdata;
static unsigned long segment_low __initdata;
static unsigned long pgalloc_pos __initdata;
static unsigned long pgalloc_low __initdata;
static unsigned long pgalloc_freeable __initdata;
static bool has_edat __initdata;
static bool has_nx __initdata;

#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))

static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);

static void __init kasan_early_panic(const char *reason)
{
	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	sclp_early_printk(reason);
	disabled_wait();
}

static void * __init kasan_early_alloc_segment(void)
{
	segment_pos -= _SEGMENT_SIZE;

	if (segment_pos < segment_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)segment_pos;
}

static void * __init kasan_early_alloc_pages(unsigned int order)
{
	pgalloc_pos -= (PAGE_SIZE << order);

	if (pgalloc_pos < pgalloc_low)
		kasan_early_panic("out of memory during initialisation\n");

	return (void *)pgalloc_pos;
}

static void * __init kasan_early_crst_alloc(unsigned long val)
{
	unsigned long *table;

	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	if (table)
		crst_table_init(table, val);
	return table;
}

static pte_t * __init kasan_early_pte_alloc(void)
{
	static void *pte_leftover;
	pte_t *pte;

	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);

	if (!pte_leftover) {
		pte_leftover = kasan_early_alloc_pages(0);
		pte = pte_leftover + _PAGE_TABLE_SIZE;
	} else {
		pte = pte_leftover;
		pte_leftover = NULL;
	}
	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	return pte;
}

enum populate_mode {
	POPULATE_ONE2ONE,
	POPULATE_MAP,
	POPULATE_ZERO_SHADOW,
	POPULATE_SHALLOW
};
static void __init kasan_early_pgtable_populate(unsigned long address,
						unsigned long end,
						enum populate_mode mode)
{
	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
	pgd_t *pg_dir;
	p4d_t *p4_dir;
	pud_t *pu_dir;
	pmd_t *pm_dir;
	pte_t *pt_dir;

	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
	if (!has_nx)
		pgt_prot_zero &= ~_PAGE_NOEXEC;
	pgt_prot = pgprot_val(PAGE_KERNEL);
	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	if (!has_nx || mode == POPULATE_ONE2ONE) {
		pgt_prot &= ~_PAGE_NOEXEC;
		sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	}

	while (address < end) {
		pg_dir = pgd_offset_k(address);
		if (pgd_none(*pg_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PGDIR_SIZE) &&
			    end - address >= PGDIR_SIZE) {
				pgd_populate(&init_mm, pg_dir,
						kasan_early_shadow_p4d);
				address = (address + PGDIR_SIZE) & PGDIR_MASK;
				continue;
			}
			p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
			pgd_populate(&init_mm, pg_dir, p4_dir);
		}

		if (mode == POPULATE_SHALLOW) {
			address = (address + P4D_SIZE) & P4D_MASK;
			continue;
		}

		p4_dir = p4d_offset(pg_dir, address);
		if (p4d_none(*p4_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, P4D_SIZE) &&
			    end - address >= P4D_SIZE) {
				p4d_populate(&init_mm, p4_dir,
						kasan_early_shadow_pud);
				address = (address + P4D_SIZE) & P4D_MASK;
				continue;
			}
			pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
			p4d_populate(&init_mm, p4_dir, pu_dir);
		}

		pu_dir = pud_offset(p4_dir, address);
		if (pud_none(*pu_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PUD_SIZE) &&
			    end - address >= PUD_SIZE) {
				pud_populate(&init_mm, pu_dir,
						kasan_early_shadow_pmd);
				address = (address + PUD_SIZE) & PUD_MASK;
				continue;
			}
			pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
			pud_populate(&init_mm, pu_dir, pm_dir);
		}

		pm_dir = pmd_offset(pu_dir, address);
		if (pmd_none(*pm_dir)) {
			if (mode == POPULATE_ZERO_SHADOW &&
			    IS_ALIGNED(address, PMD_SIZE) &&
			    end - address >= PMD_SIZE) {
				pmd_populate(&init_mm, pm_dir,
						kasan_early_shadow_pte);
				address = (address + PMD_SIZE) & PMD_MASK;
				continue;
			}
			/* the first megabyte of 1:1 is mapped with 4k pages */
			if (has_edat && address && end - address >= PMD_SIZE &&
			    mode != POPULATE_ZERO_SHADOW) {
				void *page;

				if (mode == POPULATE_ONE2ONE) {
					page = (void *)address;
				} else {
					page = kasan_early_alloc_segment();
					memset(page, 0, _SEGMENT_SIZE);
				}
				pmd_val(*pm_dir) = __pa(page) | sgt_prot;
				address = (address + PMD_SIZE) & PMD_MASK;
				continue;
			}

			pt_dir = kasan_early_pte_alloc();
			pmd_populate(&init_mm, pm_dir, pt_dir);
		} else if (pmd_large(*pm_dir)) {
			address = (address + PMD_SIZE) & PMD_MASK;
			continue;
		}

		pt_dir = pte_offset_kernel(pm_dir, address);
		if (pte_none(*pt_dir)) {
			void *page;

			switch (mode) {
			case POPULATE_ONE2ONE:
				page = (void *)address;
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_MAP:
				page = kasan_early_alloc_pages(0);
				memset(page, 0, PAGE_SIZE);
				pte_val(*pt_dir) = __pa(page) | pgt_prot;
				break;
			case POPULATE_ZERO_SHADOW:
				page = kasan_early_shadow_page;
				pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
				break;
			case POPULATE_SHALLOW:
				/* should never happen */
				break;
			}
		}
		address += PAGE_SIZE;
	}
}

static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
{
	unsigned long asce_bits;

	asce_bits = asce_type | _ASCE_TABLE_LENGTH;
	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
	S390_lowcore.user_asce = S390_lowcore.kernel_asce;

	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
}

static void __init kasan_enable_dat(void)
{
	psw_t psw;

	psw.mask = __extract_psw();
	psw_bits(psw).dat = 1;
	psw_bits(psw).as = PSW_BITS_AS_HOME;
	__load_psw_mask(psw.mask);
}

static void __init kasan_early_detect_facilities(void)
{
	if (test_facility(8)) {
		has_edat = true;
		__ctl_set_bit(0, 23);
	}
	if (!noexec_disabled && test_facility(130)) {
		has_nx = true;
		__ctl_set_bit(0, 20);
	}
}

static bool __init has_uv_sec_stor_limit(void)
{
	/*
	 * keep these conditions in line with setup_uv()
	 */
	if (!is_prot_virt_host())
		return false;

	if (is_prot_virt_guest())
		return false;

	if (!test_facility(158))
		return false;

	return !!uv_info.max_sec_stor_addr;
}

void __init kasan_early_init(void)
{
	unsigned long untracked_mem_end;
	unsigned long shadow_alloc_size;
	unsigned long vmax_unlimited;
	unsigned long initrd_end;
	unsigned long memsize;
	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
	pte_t pte_z;
	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);

	kasan_early_detect_facilities();
	if (!has_nx)
		pgt_prot &= ~_PAGE_NOEXEC;
	pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);

	memsize = get_mem_detect_end();
	if (!memsize)
		kasan_early_panic("cannot detect physical memory size\n");
	/*
	 * Kasan currently supports standby memory but only if it follows
	 * online memory (default allocation), i.e. no memory holes.
	 * - memsize represents end of online memory
	 * - ident_map_size represents online + standby and memory limits
	 *   accounted.
	 * Kasan maps "memsize" right away.
	 * [0, memsize]			- as identity mapping
	 * [__sha(0), __sha(memsize)]	- shadow memory for identity mapping
	 * The rest [memsize, ident_map_size] if memsize < ident_map_size
	 * could be mapped/unmapped dynamically later during memory hotplug.
	 */
	memsize = min(memsize, ident_map_size);

	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
	crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);
	untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
	if (has_uv_sec_stor_limit())
		kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);

	/* init kasan zero shadow */
	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
				p4d_val(p4d_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pud,
				pud_val(pud_z));
	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
				pmd_val(pmd_z));
	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);

	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
		initrd_end =
		    round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
		pgalloc_low = max(pgalloc_low, initrd_end);
	}

	if (pgalloc_low + shadow_alloc_size > memsize)
		kasan_early_panic("out of memory during initialisation\n");

	if (has_edat) {
		segment_pos = round_down(memsize, _SEGMENT_SIZE);
		segment_low = segment_pos - shadow_alloc_size;
		pgalloc_pos = segment_low;
	} else {
		pgalloc_pos = memsize;
	}
	init_mm.pgd = early_pg_dir;
	/*
	 * Current memory layout:
	 * +- 0 -------------+	   +- shadow start -+
	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
	 * |		     |	 / |		    |
	 * +- end of ram ----+	/  +----------------+
	 * | ... gap ...     | /   |		    |
	 * |		     |/    |	kasan	    |
	 * +- shadow start --+	   |	zero	    |
	 * | 1/8 addr space  |	   |	page	    |
	 * +- shadow end    -+	   |	mapping	    |
	 * | ... gap ...     |\    |  (untracked)   |
	 * +- vmalloc area  -+ \   |		    |
	 * | vmalloc_size    |	\  |		    |
	 * +- modules vaddr -+	 \ +----------------+
	 * | 2Gb	     |	  \|	  unmapped  | allocated per module
	 * +-----------------+	   +- shadow end ---+
	 *
	 * Current memory layout (KASAN_VMALLOC):
	 * +- 0 -------------+	   +- shadow start -+
	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
	 * |		     |	 / |		    |
	 * +- end of ram ----+	/  +----------------+
	 * | ... gap ...     | /   |	kasan	    |
	 * |		     |/    |	zero	    |
	 * +- shadow start --+	   |	page	    |
	 * | 1/8 addr space  |	   |	mapping     |
	 * +- shadow end    -+	   |  (untracked)   |
	 * | ... gap ...     |\    |		    |
	 * +- vmalloc area  -+ \   +- vmalloc area -+
	 * | vmalloc_size    |	\  |shallow populate|
	 * +- modules vaddr -+	 \ +- modules area -+
	 * | 2Gb	     |	  \|shallow populate|
	 * +-----------------+	   +- shadow end ---+
	 */
	/* populate kasan shadow (for identity mapping and zero page mapping) */
	kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
	if (IS_ENABLED(CONFIG_MODULES))
		untracked_mem_end = kasan_vmax - MODULES_LEN;
	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
		untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
		/* shallowly populate kasan shadow for vmalloc and modules */
		kasan_early_pgtable_populate(__sha(untracked_mem_end), __sha(kasan_vmax),
					     POPULATE_SHALLOW);
	}
	/* populate kasan shadow for untracked memory */
	kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_mem_end),
				     POPULATE_ZERO_SHADOW);
	kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited),
				     POPULATE_ZERO_SHADOW);
	/* memory allocated for identity mapping structs will be freed later */
	pgalloc_freeable = pgalloc_pos;
	/* populate identity mapping */
	kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE);
	kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2);
	kasan_enable_dat();
	/* enable kasan */
	init_task.kasan_depth = 0;
	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
	sclp_early_printk("KernelAddressSanitizer initialized\n");
}

void __init kasan_copy_shadow_mapping(void)
{
	/*
	 * At this point we are still running on early pages setup early_pg_dir,
	 * while swapper_pg_dir has just been initialized with identity mapping.
	 * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
	 */

	pgd_t *pg_dir_src;
	pgd_t *pg_dir_dst;
	p4d_t *p4_dir_src;
	p4d_t *p4_dir_dst;

	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
	pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START);
	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
	memcpy(p4_dir_dst, p4_dir_src,
	       (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
}

void __init kasan_free_early_identity(void)
{
	memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
}
Commit	Line	Data
42db5ed8 VG	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kasan.h>
	3	#include <linux/sched/task.h>
	4	#include <linux/memblock.h>
ca5999fd	5	#include <linux/pgtable.h>
65fddcfc	6	#include <asm/pgalloc.h>
42db5ed8	7	#include <asm/kasan.h>
29635239	8	#include <asm/mem_detect.h>
42db5ed8 VG	9	#include <asm/processor.h>
42db5ed8 VG	10	#include <asm/sclp.h>
d58106c3	11	#include <asm/facility.h>
42db5ed8 VG	12	#include <asm/sections.h>
42db5ed8 VG	13	#include <asm/setup.h>
c360c9a2	14	#include <asm/uv.h>
42db5ed8	15
c360c9a2	16	unsigned long kasan_vmax;
d58106c3 VG	17	static unsigned long segment_pos __initdata;
d58106c3 VG	18	static unsigned long segment_low __initdata;
42db5ed8 VG	19	static unsigned long pgalloc_pos __initdata;
42db5ed8 VG	20	static unsigned long pgalloc_low __initdata;
135ff163	21	static unsigned long pgalloc_freeable __initdata;
d58106c3 VG	22	static bool has_edat __initdata;
d58106c3 VG	23	static bool has_nx __initdata;
42db5ed8 VG	24
	25	#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
	26
	27	static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
	28
	29	static void __init kasan_early_panic(const char *reason)
	30	{
	31	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
	32	sclp_early_printk(reason);
184c460f	33	disabled_wait();
42db5ed8 VG	34	}
42db5ed8 VG	35
d58106c3 VG	36	static void * __init kasan_early_alloc_segment(void)
	37	{
	38	segment_pos -= _SEGMENT_SIZE;
	39
	40	if (segment_pos < segment_low)
	41	kasan_early_panic("out of memory during initialisation\n");
	42
	43	return (void *)segment_pos;
	44	}
	45
42db5ed8 VG	46	static void * __init kasan_early_alloc_pages(unsigned int order)
	47	{
	48	pgalloc_pos -= (PAGE_SIZE << order);
	49
	50	if (pgalloc_pos < pgalloc_low)
	51	kasan_early_panic("out of memory during initialisation\n");
	52
	53	return (void *)pgalloc_pos;
	54	}
	55
	56	static void * __init kasan_early_crst_alloc(unsigned long val)
	57	{
	58	unsigned long *table;
	59
	60	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
	61	if (table)
	62	crst_table_init(table, val);
	63	return table;
	64	}
	65
	66	static pte_t * __init kasan_early_pte_alloc(void)
	67	{
	68	static void *pte_leftover;
	69	pte_t *pte;
	70
	71	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
	72
	73	if (!pte_leftover) {
	74	pte_leftover = kasan_early_alloc_pages(0);
	75	pte = pte_leftover + _PAGE_TABLE_SIZE;
	76	} else {
	77	pte = pte_leftover;
	78	pte_leftover = NULL;
	79	}
	80	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
	81	return pte;
	82	}
	83
	84	enum populate_mode {
	85	POPULATE_ONE2ONE,
	86	POPULATE_MAP,
3e39ce26 VG	87	POPULATE_ZERO_SHADOW,
3e39ce26 VG	88	POPULATE_SHALLOW
42db5ed8	89	};
92bca2fe	90	static void __init kasan_early_pgtable_populate(unsigned long address,
42db5ed8 VG	91	unsigned long end,
	92	enum populate_mode mode)
	93	{
d58106c3	94	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
42db5ed8 VG	95	pgd_t *pg_dir;
	96	p4d_t *p4_dir;
	97	pud_t *pu_dir;
	98	pmd_t *pm_dir;
	99	pte_t *pt_dir;
	100
	101	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
d58106c3 VG	102	if (!has_nx)
d58106c3 VG	103	pgt_prot_zero &= ~_PAGE_NOEXEC;
d411e3c6 VG	104	pgt_prot = pgprot_val(PAGE_KERNEL);
	105	sgt_prot = pgprot_val(SEGMENT_KERNEL);
	106	if (!has_nx \|\| mode == POPULATE_ONE2ONE) {
	107	pgt_prot &= ~_PAGE_NOEXEC;
	108	sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
	109	}
42db5ed8 VG	110
	111	while (address < end) {
	112	pg_dir = pgd_offset_k(address);
	113	if (pgd_none(*pg_dir)) {
	114	if (mode == POPULATE_ZERO_SHADOW &&
	115	IS_ALIGNED(address, PGDIR_SIZE) &&
	116	end - address >= PGDIR_SIZE) {
9577dd74 AK	117	pgd_populate(&init_mm, pg_dir,
9577dd74 AK	118	kasan_early_shadow_p4d);
42db5ed8 VG	119	address = (address + PGDIR_SIZE) & PGDIR_MASK;
	120	continue;
	121	}
	122	p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
	123	pgd_populate(&init_mm, pg_dir, p4_dir);
	124	}
	125
a3453d92	126	if (mode == POPULATE_SHALLOW) {
3e39ce26 VG	127	address = (address + P4D_SIZE) & P4D_MASK;
	128	continue;
	129	}
	130
42db5ed8 VG	131	p4_dir = p4d_offset(pg_dir, address);
	132	if (p4d_none(*p4_dir)) {
	133	if (mode == POPULATE_ZERO_SHADOW &&
	134	IS_ALIGNED(address, P4D_SIZE) &&
	135	end - address >= P4D_SIZE) {
9577dd74 AK	136	p4d_populate(&init_mm, p4_dir,
9577dd74 AK	137	kasan_early_shadow_pud);
42db5ed8 VG	138	address = (address + P4D_SIZE) & P4D_MASK;
	139	continue;
	140	}
	141	pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
	142	p4d_populate(&init_mm, p4_dir, pu_dir);
	143	}
	144
	145	pu_dir = pud_offset(p4_dir, address);
	146	if (pud_none(*pu_dir)) {
	147	if (mode == POPULATE_ZERO_SHADOW &&
	148	IS_ALIGNED(address, PUD_SIZE) &&
	149	end - address >= PUD_SIZE) {
9577dd74 AK	150	pud_populate(&init_mm, pu_dir,
9577dd74 AK	151	kasan_early_shadow_pmd);
42db5ed8 VG	152	address = (address + PUD_SIZE) & PUD_MASK;
	153	continue;
	154	}
	155	pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
	156	pud_populate(&init_mm, pu_dir, pm_dir);
	157	}
	158
	159	pm_dir = pmd_offset(pu_dir, address);
	160	if (pmd_none(*pm_dir)) {
	161	if (mode == POPULATE_ZERO_SHADOW &&
	162	IS_ALIGNED(address, PMD_SIZE) &&
	163	end - address >= PMD_SIZE) {
9577dd74 AK	164	pmd_populate(&init_mm, pm_dir,
9577dd74 AK	165	kasan_early_shadow_pte);
42db5ed8 VG	166	address = (address + PMD_SIZE) & PMD_MASK;
	167	continue;
	168	}
d58106c3 VG	169	/* the first megabyte of 1:1 is mapped with 4k pages */
	170	if (has_edat && address && end - address >= PMD_SIZE &&
	171	mode != POPULATE_ZERO_SHADOW) {
	172	void *page;
	173
	174	if (mode == POPULATE_ONE2ONE) {
	175	page = (void *)address;
	176	} else {
	177	page = kasan_early_alloc_segment();
	178	memset(page, 0, _SEGMENT_SIZE);
	179	}
	180	pmd_val(*pm_dir) = __pa(page) \| sgt_prot;
	181	address = (address + PMD_SIZE) & PMD_MASK;
	182	continue;
	183	}
	184
42db5ed8 VG	185	pt_dir = kasan_early_pte_alloc();
42db5ed8 VG	186	pmd_populate(&init_mm, pm_dir, pt_dir);
d58106c3 VG	187	} else if (pmd_large(*pm_dir)) {
	188	address = (address + PMD_SIZE) & PMD_MASK;
	189	continue;
42db5ed8 VG	190	}
	191
	192	pt_dir = pte_offset_kernel(pm_dir, address);
	193	if (pte_none(*pt_dir)) {
	194	void *page;
	195
	196	switch (mode) {
	197	case POPULATE_ONE2ONE:
	198	page = (void *)address;
	199	pte_val(*pt_dir) = __pa(page) \| pgt_prot;
	200	break;
	201	case POPULATE_MAP:
	202	page = kasan_early_alloc_pages(0);
	203	memset(page, 0, PAGE_SIZE);
	204	pte_val(*pt_dir) = __pa(page) \| pgt_prot;
	205	break;
	206	case POPULATE_ZERO_SHADOW:
9577dd74	207	page = kasan_early_shadow_page;
42db5ed8 VG	208	pte_val(*pt_dir) = __pa(page) \| pgt_prot_zero;
42db5ed8 VG	209	break;
3e39ce26 VG	210	case POPULATE_SHALLOW:
	211	/* should never happen */
	212	break;
42db5ed8 VG	213	}
	214	}
	215	address += PAGE_SIZE;
	216	}
	217	}
	218
	219	static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
	220	{
	221	unsigned long asce_bits;
	222
	223	asce_bits = asce_type \| _ASCE_TABLE_LENGTH;
	224	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) \| asce_bits;
	225	S390_lowcore.user_asce = S390_lowcore.kernel_asce;
	226
	227	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
	228	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
	229	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
	230	}
	231
	232	static void __init kasan_enable_dat(void)
	233	{
	234	psw_t psw;
	235
	236	psw.mask = __extract_psw();
	237	psw_bits(psw).dat = 1;
	238	psw_bits(psw).as = PSW_BITS_AS_HOME;
	239	__load_psw_mask(psw.mask);
	240	}
	241
d58106c3 VG	242	static void __init kasan_early_detect_facilities(void)
d58106c3 VG	243	{
d58106c3 VG	244	if (test_facility(8)) {
	245	has_edat = true;
	246	__ctl_set_bit(0, 23);
	247	}
	248	if (!noexec_disabled && test_facility(130)) {
	249	has_nx = true;
	250	__ctl_set_bit(0, 20);
	251	}
	252	}
	253
c360c9a2 VG	254	static bool __init has_uv_sec_stor_limit(void)
	255	{
	256	/*
	257	* keep these conditions in line with setup_uv()
	258	*/
	259	if (!is_prot_virt_host())
	260	return false;
	261
	262	if (is_prot_virt_guest())
	263	return false;
	264
	265	if (!test_facility(158))
	266	return false;
	267
	268	return !!uv_info.max_sec_stor_addr;
	269	}
	270
42db5ed8 VG	271	void __init kasan_early_init(void)
	272	{
	273	unsigned long untracked_mem_end;
	274	unsigned long shadow_alloc_size;
c360c9a2	275	unsigned long vmax_unlimited;
42db5ed8	276	unsigned long initrd_end;
42db5ed8	277	unsigned long memsize;
42db5ed8 VG	278	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
42db5ed8 VG	279	pte_t pte_z;
9577dd74 AK	280	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) \| _SEGMENT_ENTRY);
	281	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) \| _REGION3_ENTRY);
	282	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) \| _REGION2_ENTRY);
42db5ed8	283
d58106c3 VG	284	kasan_early_detect_facilities();
	285	if (!has_nx)
	286	pgt_prot &= ~_PAGE_NOEXEC;
9577dd74	287	pte_z = __pte(__pa(kasan_early_shadow_page) \| pgt_prot);
42db5ed8	288
29635239 VG	289	memsize = get_mem_detect_end();
	290	if (!memsize)
	291	kasan_early_panic("cannot detect physical memory size\n");
73045a08 VG	292	/*
	293	* Kasan currently supports standby memory but only if it follows
	294	* online memory (default allocation), i.e. no memory holes.
	295	* - memsize represents end of online memory
	296	* - ident_map_size represents online + standby and memory limits
	297	* accounted.
	298	* Kasan maps "memsize" right away.
	299	* [0, memsize] - as identity mapping
	300	* [__sha(0), __sha(memsize)] - shadow memory for identity mapping
	301	* The rest [memsize, ident_map_size] if memsize < ident_map_size
	302	* could be mapped/unmapped dynamically later during memory hotplug.
	303	*/
	304	memsize = min(memsize, ident_map_size);
29635239	305
a3453d92 VG	306	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
	307	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
	308	crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);
	309	untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
	310	if (has_uv_sec_stor_limit())
	311	kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
42db5ed8 VG	312
42db5ed8 VG	313	/* init kasan zero shadow */
9577dd74 AK	314	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
	315	p4d_val(p4d_z));
	316	crst_table_init((unsigned long *)kasan_early_shadow_pud,
	317	pud_val(pud_z));
	318	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
	319	pmd_val(pmd_z));
	320	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
42db5ed8	321
42db5ed8	322	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
42db5ed8 VG	323	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
	324	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
	325	initrd_end =
	326	round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
	327	pgalloc_low = max(pgalloc_low, initrd_end);
	328	}
	329
	330	if (pgalloc_low + shadow_alloc_size > memsize)
	331	kasan_early_panic("out of memory during initialisation\n");
	332
d58106c3 VG	333	if (has_edat) {
	334	segment_pos = round_down(memsize, _SEGMENT_SIZE);
	335	segment_low = segment_pos - shadow_alloc_size;
	336	pgalloc_pos = segment_low;
	337	} else {
	338	pgalloc_pos = memsize;
	339	}
42db5ed8 VG	340	init_mm.pgd = early_pg_dir;
	341	/*
	342	* Current memory layout:
3e39ce26 VG	343	* +- 0 -------------+ +- shadow start -+
	344	* \| 1:1 ram mapping \| /\| 1/8 ram \|
	345	* \| \| / \| \|
	346	* +- end of ram ----+ / +----------------+
	347	* \| ... gap ... \| / \| \|
	348	* \| \|/ \| kasan \|
	349	* +- shadow start --+ \| zero \|
	350	* \| 1/8 addr space \| \| page \|
	351	* +- shadow end -+ \| mapping \|
	352	* \| ... gap ... \|\ \| (untracked) \|
	353	* +- vmalloc area -+ \ \| \|
	354	* \| vmalloc_size \| \ \| \|
	355	* +- modules vaddr -+ \ +----------------+
	356	* \| 2Gb \| \\| unmapped \| allocated per module
	357	* +-----------------+ +- shadow end ---+
	358	*
	359	* Current memory layout (KASAN_VMALLOC):
	360	* +- 0 -------------+ +- shadow start -+
	361	* \| 1:1 ram mapping \| /\| 1/8 ram \|
	362	* \| \| / \| \|
	363	* +- end of ram ----+ / +----------------+
	364	* \| ... gap ... \| / \| kasan \|
	365	* \| \|/ \| zero \|
	366	* +- shadow start --+ \| page \|
	367	* \| 1/8 addr space \| \| mapping \|
	368	* +- shadow end -+ \| (untracked) \|
	369	* \| ... gap ... \|\ \| \|
	370	* +- vmalloc area -+ \ +- vmalloc area -+
	371	* \| vmalloc_size \| \ \|shallow populate\|
	372	* +- modules vaddr -+ \ +- modules area -+
	373	* \| 2Gb \| \\|shallow populate\|
	374	* +-----------------+ +- shadow end ---+
42db5ed8	375	*/
793213a8	376	/* populate kasan shadow (for identity mapping and zero page mapping) */
92bca2fe	377	kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
793213a8	378	if (IS_ENABLED(CONFIG_MODULES))
c360c9a2	379	untracked_mem_end = kasan_vmax - MODULES_LEN;
3e39ce26	380	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
c360c9a2	381	untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
3e39ce26	382	/* shallowly populate kasan shadow for vmalloc and modules */
92bca2fe VG	383	kasan_early_pgtable_populate(__sha(untracked_mem_end), __sha(kasan_vmax),
92bca2fe VG	384	POPULATE_SHALLOW);
3e39ce26 VG	385	}
3e39ce26 VG	386	/* populate kasan shadow for untracked memory */
73045a08	387	kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_mem_end),
42db5ed8	388	POPULATE_ZERO_SHADOW);
92bca2fe	389	kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited),
c360c9a2	390	POPULATE_ZERO_SHADOW);
135ff163 VG	391	/* memory allocated for identity mapping structs will be freed later */
	392	pgalloc_freeable = pgalloc_pos;
	393	/* populate identity mapping */
92bca2fe	394	kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE);
a3453d92	395	kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2);
42db5ed8 VG	396	kasan_enable_dat();
	397	/* enable kasan */
	398	init_task.kasan_depth = 0;
	399	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
	400	sclp_early_printk("KernelAddressSanitizer initialized\n");
	401	}
	402
54b52981	403	void __init kasan_copy_shadow_mapping(void)
42db5ed8 VG	404	{
	405	/*
	406	* At this point we are still running on early pages setup early_pg_dir,
	407	* while swapper_pg_dir has just been initialized with identity mapping.
	408	* Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
	409	*/
	410
	411	pgd_t *pg_dir_src;
	412	pgd_t *pg_dir_dst;
	413	p4d_t *p4_dir_src;
	414	p4d_t *p4_dir_dst;
42db5ed8 VG	415
42db5ed8 VG	416	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
54b52981	417	pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START);
42db5ed8 VG	418	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
42db5ed8 VG	419	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
a3453d92 VG	420	memcpy(p4_dir_dst, p4_dir_src,
a3453d92 VG	421	(KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
42db5ed8	422	}
135ff163 VG	423
	424	void __init kasan_free_early_identity(void)
	425	{
	426	memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
	427	}