arch/powerpc/kernel/vdso.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2
   3 /*
   4  *    Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
   5  *                       <[email protected]>
   6  */
   7
   8 #include <linux/errno.h>
   9 #include <linux/sched.h>
  10 #include <linux/kernel.h>
  11 #include <linux/mm.h>
  12 #include <linux/smp.h>
  13 #include <linux/stddef.h>
  14 #include <linux/unistd.h>
  15 #include <linux/slab.h>
  16 #include <linux/user.h>
  17 #include <linux/elf.h>
  18 #include <linux/security.h>
  19 #include <linux/syscalls.h>
  20 #include <linux/time_namespace.h>
  21 #include <vdso/datapage.h>
  22
  23 #include <asm/syscall.h>
  24 #include <asm/processor.h>
  25 #include <asm/mmu.h>
  26 #include <asm/mmu_context.h>
  27 #include <asm/machdep.h>
  28 #include <asm/cputable.h>
  29 #include <asm/sections.h>
  30 #include <asm/firmware.h>
  31 #include <asm/vdso.h>
  32 #include <asm/vdso_datapage.h>
  33 #include <asm/setup.h>
  34
  35 /* The alignment of the vDSO */
  36 #define VDSO_ALIGNMENT  (1 << 16)
  37
  38 extern char vdso32_start, vdso32_end;
  39 extern char vdso64_start, vdso64_end;
  40
  41 long sys_ni_syscall(void);
  42
  43 /*
  44  * The vdso data page (aka. systemcfg for old ppc64 fans) is here.
  45  * Once the early boot kernel code no longer needs to muck around
  46  * with it, it will become dynamically allocated
  47  */
  48 static union {
  49         struct vdso_arch_data   data;
  50         u8                      page[2 * PAGE_SIZE];
  51 } vdso_data_store __page_aligned_data;
  52 struct vdso_arch_data *vdso_data = &vdso_data_store.data;
  53
  54 enum vvar_pages {
  55         VVAR_BASE_PAGE_OFFSET,
  56         VVAR_TIME_PAGE_OFFSET,
  57         VVAR_TIMENS_PAGE_OFFSET,
  58         VVAR_NR_PAGES,
  59 };
  60
  61 static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma,
  62                        unsigned long text_size)
  63 {
  64         unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
  65
  66         if (new_size != text_size)
  67                 return -EINVAL;
  68
  69         current->mm->context.vdso = (void __user *)new_vma->vm_start;
  70
  71         return 0;
  72 }
  73
  74 static int vdso32_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
  75 {
  76         return vdso_mremap(sm, new_vma, &vdso32_end - &vdso32_start);
  77 }
  78
  79 static int vdso64_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
  80 {
  81         return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start);
  82 }
  83
  84 static void vdso_close(const struct vm_special_mapping *sm, struct vm_area_struct *vma)
  85 {
  86         struct mm_struct *mm = vma->vm_mm;
  87
  88         /*
  89          * close() is called for munmap() but also for mremap(). In the mremap()
  90          * case the vdso pointer has already been updated by the mremap() hook
  91          * above, so it must not be set to NULL here.
  92          */
  93         if (vma->vm_start != (unsigned long)mm->context.vdso)
  94                 return;
  95
  96         mm->context.vdso = NULL;
  97 }
  98
  99 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
 100                              struct vm_area_struct *vma, struct vm_fault *vmf);
 101
 102 static struct vm_special_mapping vvar_spec __ro_after_init = {
 103         .name = "[vvar]",
 104         .fault = vvar_fault,
 105 };
 106
 107 static struct vm_special_mapping vdso32_spec __ro_after_init = {
 108         .name = "[vdso]",
 109         .mremap = vdso32_mremap,
 110         .close = vdso_close,
 111 };
 112
 113 static struct vm_special_mapping vdso64_spec __ro_after_init = {
 114         .name = "[vdso]",
 115         .mremap = vdso64_mremap,
 116         .close = vdso_close,
 117 };
 118
 119 #ifdef CONFIG_TIME_NS
 120 struct vdso_data *arch_get_vdso_data(void *vvar_page)
 121 {
 122         return vvar_page;
 123 }
 124
 125 /*
 126  * The vvar mapping contains data for a specific time namespace, so when a task
 127  * changes namespace we must unmap its vvar data for the old namespace.
 128  * Subsequent faults will map in data for the new namespace.
 129  *
 130  * For more details see timens_setup_vdso_data().
 131  */
 132 int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
 133 {
 134         struct mm_struct *mm = task->mm;
 135         VMA_ITERATOR(vmi, mm, 0);
 136         struct vm_area_struct *vma;
 137
 138         mmap_read_lock(mm);
 139         for_each_vma(vmi, vma) {
 140                 if (vma_is_special_mapping(vma, &vvar_spec))
 141                         zap_vma_pages(vma);
 142         }
 143         mmap_read_unlock(mm);
 144
 145         return 0;
 146 }
 147 #endif
 148
 149 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
 150                              struct vm_area_struct *vma, struct vm_fault *vmf)
 151 {
 152         struct page *timens_page = find_timens_vvar_page(vma);
 153         unsigned long pfn;
 154
 155         switch (vmf->pgoff) {
 156         case VVAR_BASE_PAGE_OFFSET:
 157                 pfn = virt_to_pfn(vdso_data);
 158                 break;
 159         case VVAR_TIME_PAGE_OFFSET:
 160                 if (timens_page)
 161                         pfn = page_to_pfn(timens_page);
 162                 else
 163                         pfn = virt_to_pfn(vdso_data->data);
 164                 break;
 165 #ifdef CONFIG_TIME_NS
 166         case VVAR_TIMENS_PAGE_OFFSET:
 167                 /*
 168                  * If a task belongs to a time namespace then a namespace
 169                  * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
 170                  * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
 171                  * offset.
 172                  * See also the comment near timens_setup_vdso_data().
 173                  */
 174                 if (!timens_page)
 175                         return VM_FAULT_SIGBUS;
 176                 pfn = virt_to_pfn(vdso_data->data);
 177                 break;
 178 #endif /* CONFIG_TIME_NS */
 179         default:
 180                 return VM_FAULT_SIGBUS;
 181         }
 182
 183         return vmf_insert_pfn(vma, vmf->address, pfn);
 184 }
 185
 186 /*
 187  * This is called from binfmt_elf, we create the special vma for the
 188  * vDSO and insert it into the mm struct tree
 189  */
 190 static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 191 {
 192         unsigned long vdso_size, vdso_base, mappings_size;
 193         struct vm_special_mapping *vdso_spec;
 194         unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE;
 195         struct mm_struct *mm = current->mm;
 196         struct vm_area_struct *vma;
 197
 198         if (is_32bit_task()) {
 199                 vdso_spec = &vdso32_spec;
 200                 vdso_size = &vdso32_end - &vdso32_start;
 201         } else {
 202                 vdso_spec = &vdso64_spec;
 203                 vdso_size = &vdso64_end - &vdso64_start;
 204         }
 205
 206         mappings_size = vdso_size + vvar_size;
 207         mappings_size += (VDSO_ALIGNMENT - 1) & PAGE_MASK;
 208
 209         /*
 210          * Pick a base address for the vDSO in process space.
 211          * Add enough to the size so that the result can be aligned.
 212          */
 213         vdso_base = get_unmapped_area(NULL, 0, mappings_size, 0, 0);
 214         if (IS_ERR_VALUE(vdso_base))
 215                 return vdso_base;
 216
 217         /* Add required alignment. */
 218         vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
 219
 220         vma = _install_special_mapping(mm, vdso_base, vvar_size,
 221                                        VM_READ | VM_MAYREAD | VM_IO |
 222                                        VM_DONTDUMP | VM_PFNMAP, &vvar_spec);
 223         if (IS_ERR(vma))
 224                 return PTR_ERR(vma);
 225
 226         /*
 227          * our vma flags don't have VM_WRITE so by default, the process isn't
 228          * allowed to write those pages.
 229          * gdb can break that with ptrace interface, and thus trigger COW on
 230          * those pages but it's then your responsibility to never do that on
 231          * the "data" page of the vDSO or you'll stop getting kernel updates
 232          * and your nice userland gettimeofday will be totally dead.
 233          * It's fine to use that for setting breakpoints in the vDSO code
 234          * pages though.
 235          */
 236         vma = _install_special_mapping(mm, vdso_base + vvar_size, vdso_size,
 237                                        VM_READ | VM_EXEC | VM_MAYREAD |
 238                                        VM_MAYWRITE | VM_MAYEXEC, vdso_spec);
 239         if (IS_ERR(vma)) {
 240                 do_munmap(mm, vdso_base, vvar_size, NULL);
 241                 return PTR_ERR(vma);
 242         }
 243
 244         // Now that the mappings are in place, set the mm VDSO pointer
 245         mm->context.vdso = (void __user *)vdso_base + vvar_size;
 246
 247         return 0;
 248 }
 249
 250 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 251 {
 252         struct mm_struct *mm = current->mm;
 253         int rc;
 254
 255         mm->context.vdso = NULL;
 256
 257         if (mmap_write_lock_killable(mm))
 258                 return -EINTR;
 259
 260         rc = __arch_setup_additional_pages(bprm, uses_interp);
 261
 262         mmap_write_unlock(mm);
 263         return rc;
 264 }
 265
 266 #define VDSO_DO_FIXUPS(type, value, bits, sec) do {                                     \
 267         void *__start = (void *)VDSO##bits##_SYMBOL(&vdso##bits##_start, sec##_start);  \
 268         void *__end = (void *)VDSO##bits##_SYMBOL(&vdso##bits##_start, sec##_end);      \
 269                                                                                         \
 270         do_##type##_fixups((value), __start, __end);                                    \
 271 } while (0)
 272
 273 static void __init vdso_fixup_features(void)
 274 {
 275 #ifdef CONFIG_PPC64
 276         VDSO_DO_FIXUPS(feature, cur_cpu_spec->cpu_features, 64, ftr_fixup);
 277         VDSO_DO_FIXUPS(feature, cur_cpu_spec->mmu_features, 64, mmu_ftr_fixup);
 278         VDSO_DO_FIXUPS(feature, powerpc_firmware_features, 64, fw_ftr_fixup);
 279         VDSO_DO_FIXUPS(lwsync, cur_cpu_spec->cpu_features, 64, lwsync_fixup);
 280 #endif /* CONFIG_PPC64 */
 281
 282 #ifdef CONFIG_VDSO32
 283         VDSO_DO_FIXUPS(feature, cur_cpu_spec->cpu_features, 32, ftr_fixup);
 284         VDSO_DO_FIXUPS(feature, cur_cpu_spec->mmu_features, 32, mmu_ftr_fixup);
 285 #ifdef CONFIG_PPC64
 286         VDSO_DO_FIXUPS(feature, powerpc_firmware_features, 32, fw_ftr_fixup);
 287 #endif /* CONFIG_PPC64 */
 288         VDSO_DO_FIXUPS(lwsync, cur_cpu_spec->cpu_features, 32, lwsync_fixup);
 289 #endif
 290 }
 291
 292 /*
 293  * Called from setup_arch to initialize the bitmap of available
 294  * syscalls in the systemcfg page
 295  */
 296 static void __init vdso_setup_syscall_map(void)
 297 {
 298         unsigned int i;
 299
 300         for (i = 0; i < NR_syscalls; i++) {
 301                 if (sys_call_table[i] != (void *)&sys_ni_syscall)
 302                         vdso_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
 303                 if (IS_ENABLED(CONFIG_COMPAT) &&
 304                     compat_sys_call_table[i] != (void *)&sys_ni_syscall)
 305                         vdso_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
 306         }
 307 }
 308
 309 #ifdef CONFIG_PPC64
 310 int vdso_getcpu_init(void)
 311 {
 312         unsigned long cpu, node, val;
 313
 314         /*
 315          * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node
 316          * in the next 16 bits.  The VDSO uses this to implement getcpu().
 317          */
 318         cpu = get_cpu();
 319         WARN_ON_ONCE(cpu > 0xffff);
 320
 321         node = cpu_to_node(cpu);
 322         WARN_ON_ONCE(node > 0xffff);
 323
 324         val = (cpu & 0xffff) | ((node & 0xffff) << 16);
 325         mtspr(SPRN_SPRG_VDSO_WRITE, val);
 326         get_paca()->sprg_vdso = val;
 327
 328         put_cpu();
 329
 330         return 0;
 331 }
 332 /* We need to call this before SMP init */
 333 early_initcall(vdso_getcpu_init);
 334 #endif
 335
 336 static struct page ** __init vdso_setup_pages(void *start, void *end)
 337 {
 338         int i;
 339         struct page **pagelist;
 340         int pages = (end - start) >> PAGE_SHIFT;
 341
 342         pagelist = kcalloc(pages + 1, sizeof(struct page *), GFP_KERNEL);
 343         if (!pagelist)
 344                 panic("%s: Cannot allocate page list for VDSO", __func__);
 345
 346         for (i = 0; i < pages; i++)
 347                 pagelist[i] = virt_to_page(start + i * PAGE_SIZE);
 348
 349         return pagelist;
 350 }
 351
 352 static int __init vdso_init(void)
 353 {
 354 #ifdef CONFIG_PPC64
 355         vdso_data->dcache_block_size = ppc64_caches.l1d.block_size;
 356         vdso_data->icache_block_size = ppc64_caches.l1i.block_size;
 357         vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
 358         vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size;
 359 #endif /* CONFIG_PPC64 */
 360
 361         vdso_setup_syscall_map();
 362
 363         vdso_fixup_features();
 364
 365         if (IS_ENABLED(CONFIG_VDSO32))
 366                 vdso32_spec.pages = vdso_setup_pages(&vdso32_start, &vdso32_end);
 367
 368         if (IS_ENABLED(CONFIG_PPC64))
 369                 vdso64_spec.pages = vdso_setup_pages(&vdso64_start, &vdso64_end);
 370
 371         smp_wmb();
 372
 373         return 0;
 374 }
 375 arch_initcall(vdso_init);