x86/kaslr: Expose and use the end of the physical memory address space

iounmap() on x86 occasionally fails to unmap because the provided valid
ioremap address is not below high_memory. It turned out that this
happens due to KASLR.

KASLR uses the full address space between PAGE_OFFSET and vaddr_end to
randomize the starting points of the direct map, vmalloc and vmemmap
regions.  It thereby limits the size of the direct map by using the
installed memory size plus an extra configurable margin for hot-plug
memory.  This limitation is done to gain more randomization space
because otherwise only the holes between the direct map, vmalloc,
vmemmap and vaddr_end would be usable for randomizing.

The limited direct map size is not exposed to the rest of the kernel, so
the memory hot-plug and resource management related code paths still
operate under the assumption that the available address space can be
determined with MAX_PHYSMEM_BITS.

request_free_mem_region() allocates from (1 << MAX_PHYSMEM_BITS) - 1
downwards.  That means the first allocation happens past the end of the
direct map and if unlucky this address is in the vmalloc space, which
causes high_memory to become greater than VMALLOC_START and consequently
causes iounmap() to fail for valid ioremap addresses.

MAX_PHYSMEM_BITS cannot be changed for that because the randomization
does not align with address bit boundaries and there are other places
which actually require to know the maximum number of address bits.  All
remaining usage sites of MAX_PHYSMEM_BITS have been analyzed and found
to be correct.

Cure this by exposing the end of the direct map via PHYSMEM_END and use
that for the memory hot-plug and resource management related places
instead of relying on MAX_PHYSMEM_BITS. In the KASLR case PHYSMEM_END
maps to a variable which is initialized by the KASLR initialization and
otherwise it is based on MAX_PHYSMEM_BITS as before.

To prevent future hickups add a check into add_pages() to catch callers
trying to add memory above PHYSMEM_END.

Fixes: 0483e1fa6e09 ("x86/mm: Implement ASLR for kernel memory regions")
Reported-by: Max Ramanouski <[email protected]>
Reported-by: Alistair Popple <[email protected]>
Signed-off-by: Thomas Gleixner <[email protected]>
Tested-By: Max Ramanouski <[email protected]>
Tested-by: Alistair Popple <[email protected]>
Reviewed-by: Dan Williams <[email protected]>
Reviewed-by: Alistair Popple <[email protected]>
Reviewed-by: Kees Cook <[email protected]>
Cc: [email protected]
Link: https://lore.kernel.org/all/87ed6soy3z.ffs@tglx

diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index af4302d79b59bd05ea510f8c187c8192604fee47..f3d257c452254dd564146314c8a668afb9454d6b 100644 (file)
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -17,6 +17,7 @@ extern unsigned long phys_base;
 extern unsigned long page_offset_base;
 extern unsigned long vmalloc_base;
 extern unsigned long vmemmap_base;
+extern unsigned long physmem_end;
 
 static __always_inline unsigned long __phys_addr_nodebug(unsigned long x)
 {

diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 9053dfe9fa03f947f0ad28c20eb6bfd45856f7d3..a98e53491a4e6e6d888ac3037d24a6f31ff20cb1 100644 (file)
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -140,6 +140,10 @@ extern unsigned int ptrs_per_p4d;
 # define VMEMMAP_START         __VMEMMAP_BASE_L4
 #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 
+#ifdef CONFIG_RANDOMIZE_MEMORY
+# define PHYSMEM_END           physmem_end
+#endif
+
 /*
  * End of the region for which vmalloc page tables are pre-allocated.
  * For non-KMSAN builds, this is the same as VMALLOC_END.

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index d8dbeac8b206ddde3964542eaf0bc1d6f992f4ac..ff253648706fa9cd49169a54882014a72ad540cf 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -958,8 +958,12 @@ static void update_end_of_memory_vars(u64 start, u64 size)
 int add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages,
              struct mhp_params *params)
 {
+       unsigned long end = ((start_pfn + nr_pages) << PAGE_SHIFT) - 1;
        int ret;
 
+       if (WARN_ON_ONCE(end > PHYSMEM_END))
+               return -ERANGE;
+
        ret = __add_pages(nid, start_pfn, nr_pages, params);
        WARN_ON_ONCE(ret);
 

diff --git a/arch/x86/mm/kaslr.c b/arch/x86/mm/kaslr.c
index 37db264866b6489ef99325d2fe1604d6cd3c9885..230f1dee4f0954de03aa4432296d114e0733f5b4 100644 (file)
--- a/arch/x86/mm/kaslr.c
+++ b/arch/x86/mm/kaslr.c
@@ -47,13 +47,24 @@ static const unsigned long vaddr_end = CPU_ENTRY_AREA_BASE;
  */
 static __initdata struct kaslr_memory_region {
        unsigned long *base;
+       unsigned long *end;
        unsigned long size_tb;
 } kaslr_regions[] = {
-       { &page_offset_base, 0 },
-       { &vmalloc_base, 0 },
-       { &vmemmap_base, 0 },
+       {
+               .base   = &page_offset_base,
+               .end    = &physmem_end,
+       },
+       {
+               .base   = &vmalloc_base,
+       },
+       {
+               .base   = &vmemmap_base,
+       },
 };
 
+/* The end of the possible address space for physical memory */
+unsigned long physmem_end __ro_after_init;
+
 /* Get size in bytes used by the memory region */
 static inline unsigned long get_padding(struct kaslr_memory_region *region)
 {
@@ -82,6 +93,8 @@ void __init kernel_randomize_memory(void)
        BUILD_BUG_ON(vaddr_end != CPU_ENTRY_AREA_BASE);
        BUILD_BUG_ON(vaddr_end > __START_KERNEL_map);
 
+       /* Preset the end of the possible address space for physical memory */
+       physmem_end = ((1ULL << MAX_PHYSMEM_BITS) - 1);
        if (!kaslr_memory_enabled())
                return;
 
@@ -128,11 +141,18 @@ void __init kernel_randomize_memory(void)
                vaddr += entropy;
                *kaslr_regions[i].base = vaddr;
 
+               /* Calculate the end of the region */
+               vaddr += get_padding(&kaslr_regions[i]);
                /*
-                * Jump the region and add a minimum padding based on
-                * randomization alignment.
+                * KASLR trims the maximum possible size of the
+                * direct-map. Update the physmem_end boundary.
+                * No rounding required as the region starts
+                * PUD aligned and size is in units of TB.
                 */
-               vaddr += get_padding(&kaslr_regions[i]);
+               if (kaslr_regions[i].end)
+                       *kaslr_regions[i].end = __pa_nodebug(vaddr - 1);
+
+               /* Add a minimum padding based on randomization alignment. */
                vaddr = round_up(vaddr + 1, PUD_SIZE);
                remain_entropy -= entropy;
        }

diff --git a/include/linux/mm.h b/include/linux/mm.h
index c4b238a20b76e906c4a29fe58cb76cb5e95a56a3..b3864156eaa4e1de3c3cf6f31e8c096b5adf1662 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -97,6 +97,10 @@ extern const int mmap_rnd_compat_bits_max;
 extern int mmap_rnd_compat_bits __read_mostly;
 #endif
 
+#ifndef PHYSMEM_END
+# define PHYSMEM_END   ((1ULL << MAX_PHYSMEM_BITS) - 1)
+#endif
+
 #include <asm/page.h>
 #include <asm/processor.h>
 

diff --git a/kernel/resource.c b/kernel/resource.c
index 14777afb0a99e84820315a652157ff73892e9aeb..a83040fde236fba1339a5ecab58d15295d5396a8 100644 (file)
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -1826,8 +1826,7 @@ static resource_size_t gfr_start(struct resource *base, resource_size_t size,
        if (flags & GFR_DESCENDING) {
                resource_size_t end;
 
-               end = min_t(resource_size_t, base->end,
-                           (1ULL << MAX_PHYSMEM_BITS) - 1);
+               end = min_t(resource_size_t, base->end, PHYSMEM_END);
                return end - size + 1;
        }
 
@@ -1844,8 +1843,7 @@ static bool gfr_continue(struct resource *base, resource_size_t addr,
         * @size did not wrap 0.
         */
        return addr > addr - size &&
-              addr <= min_t(resource_size_t, base->end,
-                            (1ULL << MAX_PHYSMEM_BITS) - 1);
+              addr <= min_t(resource_size_t, base->end, PHYSMEM_END);
 }
 
 static resource_size_t gfr_next(resource_size_t addr, resource_size_t size,

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 66267c26ca1bbff3bf3821a90a371161901f3ae1..951878ab627a87fad1fb9647079f320badd09a8b 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1681,7 +1681,7 @@ struct range __weak arch_get_mappable_range(void)
 
 struct range mhp_get_pluggable_range(bool need_mapping)
 {
-       const u64 max_phys = (1ULL << MAX_PHYSMEM_BITS) - 1;
+       const u64 max_phys = PHYSMEM_END;
        struct range mhp_range;
 
        if (need_mapping) {

diff --git a/mm/sparse.c b/mm/sparse.c
index e4b830091d137387c1a18536e494a8f2db8f8c58..0c3bff882033c5f4873485590918e143c2a61010 100644 (file)
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -129,7 +129,7 @@ static inline int sparse_early_nid(struct mem_section *section)
 static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
                                                unsigned long *end_pfn)
 {
-       unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT);
+       unsigned long max_sparsemem_pfn = (PHYSMEM_END + 1) >> PAGE_SHIFT;
 
        /*
         * Sanity checks - do not allow an architecture to pass