arch/x86/kernel/machine_kexec_64.c

   1 /*
   2  * handle transition of Linux booting another kernel
   3  * Copyright (C) 2002-2005 Eric Biederman  <[email protected]>
   4  *
   5  * This source code is licensed under the GNU General Public License,
   6  * Version 2.  See the file COPYING for more details.
   7  */
   8
   9 #define pr_fmt(fmt)     "kexec: " fmt
  10
  11 #include <linux/mm.h>
  12 #include <linux/kexec.h>
  13 #include <linux/string.h>
  14 #include <linux/gfp.h>
  15 #include <linux/reboot.h>
  16 #include <linux/numa.h>
  17 #include <linux/ftrace.h>
  18 #include <linux/io.h>
  19 #include <linux/suspend.h>
  20 #include <linux/vmalloc.h>
  21
  22 #include <asm/init.h>
  23 #include <asm/pgtable.h>
  24 #include <asm/tlbflush.h>
  25 #include <asm/mmu_context.h>
  26 #include <asm/io_apic.h>
  27 #include <asm/debugreg.h>
  28 #include <asm/kexec-bzimage64.h>
  29 #include <asm/setup.h>
  30 #include <asm/set_memory.h>
  31
  32 #ifdef CONFIG_KEXEC_FILE
  33 const struct kexec_file_ops * const kexec_file_loaders[] = {
  34                 &kexec_bzImage64_ops,
  35                 NULL
  36 };
  37 #endif
  38
  39 static void free_transition_pgtable(struct kimage *image)
  40 {
  41         free_page((unsigned long)image->arch.p4d);
  42         image->arch.p4d = NULL;
  43         free_page((unsigned long)image->arch.pud);
  44         image->arch.pud = NULL;
  45         free_page((unsigned long)image->arch.pmd);
  46         image->arch.pmd = NULL;
  47         free_page((unsigned long)image->arch.pte);
  48         image->arch.pte = NULL;
  49 }
  50
  51 static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
  52 {
  53         p4d_t *p4d;
  54         pud_t *pud;
  55         pmd_t *pmd;
  56         pte_t *pte;
  57         unsigned long vaddr, paddr;
  58         int result = -ENOMEM;
  59
  60         vaddr = (unsigned long)relocate_kernel;
  61         paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE);
  62         pgd += pgd_index(vaddr);
  63         if (!pgd_present(*pgd)) {
  64                 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
  65                 if (!p4d)
  66                         goto err;
  67                 image->arch.p4d = p4d;
  68                 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
  69         }
  70         p4d = p4d_offset(pgd, vaddr);
  71         if (!p4d_present(*p4d)) {
  72                 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
  73                 if (!pud)
  74                         goto err;
  75                 image->arch.pud = pud;
  76                 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
  77         }
  78         pud = pud_offset(p4d, vaddr);
  79         if (!pud_present(*pud)) {
  80                 pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
  81                 if (!pmd)
  82                         goto err;
  83                 image->arch.pmd = pmd;
  84                 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
  85         }
  86         pmd = pmd_offset(pud, vaddr);
  87         if (!pmd_present(*pmd)) {
  88                 pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
  89                 if (!pte)
  90                         goto err;
  91                 image->arch.pte = pte;
  92                 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
  93         }
  94         pte = pte_offset_kernel(pmd, vaddr);
  95         set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC_NOENC));
  96         return 0;
  97 err:
  98         return result;
  99 }
 100
 101 static void *alloc_pgt_page(void *data)
 102 {
 103         struct kimage *image = (struct kimage *)data;
 104         struct page *page;
 105         void *p = NULL;
 106
 107         page = kimage_alloc_control_pages(image, 0);
 108         if (page) {
 109                 p = page_address(page);
 110                 clear_page(p);
 111         }
 112
 113         return p;
 114 }
 115
 116 static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
 117 {
 118         struct x86_mapping_info info = {
 119                 .alloc_pgt_page = alloc_pgt_page,
 120                 .context        = image,
 121                 .page_flag      = __PAGE_KERNEL_LARGE_EXEC,
 122                 .kernpg_flag    = _KERNPG_TABLE_NOENC,
 123         };
 124         unsigned long mstart, mend;
 125         pgd_t *level4p;
 126         int result;
 127         int i;
 128
 129         level4p = (pgd_t *)__va(start_pgtable);
 130         clear_page(level4p);
 131
 132         if (direct_gbpages)
 133                 info.direct_gbpages = true;
 134
 135         for (i = 0; i < nr_pfn_mapped; i++) {
 136                 mstart = pfn_mapped[i].start << PAGE_SHIFT;
 137                 mend   = pfn_mapped[i].end << PAGE_SHIFT;
 138
 139                 result = kernel_ident_mapping_init(&info,
 140                                                  level4p, mstart, mend);
 141                 if (result)
 142                         return result;
 143         }
 144
 145         /*
 146          * segments's mem ranges could be outside 0 ~ max_pfn,
 147          * for example when jump back to original kernel from kexeced kernel.
 148          * or first kernel is booted with user mem map, and second kernel
 149          * could be loaded out of that range.
 150          */
 151         for (i = 0; i < image->nr_segments; i++) {
 152                 mstart = image->segment[i].mem;
 153                 mend   = mstart + image->segment[i].memsz;
 154
 155                 result = kernel_ident_mapping_init(&info,
 156                                                  level4p, mstart, mend);
 157
 158                 if (result)
 159                         return result;
 160         }
 161
 162         return init_transition_pgtable(image, level4p);
 163 }
 164
 165 static void set_idt(void *newidt, u16 limit)
 166 {
 167         struct desc_ptr curidt;
 168
 169         /* x86-64 supports unaliged loads & stores */
 170         curidt.size    = limit;
 171         curidt.address = (unsigned long)newidt;
 172
 173         __asm__ __volatile__ (
 174                 "lidtq %0\n"
 175                 : : "m" (curidt)
 176                 );
 177 };
 178
 179
 180 static void set_gdt(void *newgdt, u16 limit)
 181 {
 182         struct desc_ptr curgdt;
 183
 184         /* x86-64 supports unaligned loads & stores */
 185         curgdt.size    = limit;
 186         curgdt.address = (unsigned long)newgdt;
 187
 188         __asm__ __volatile__ (
 189                 "lgdtq %0\n"
 190                 : : "m" (curgdt)
 191                 );
 192 };
 193
 194 static void load_segments(void)
 195 {
 196         __asm__ __volatile__ (
 197                 "\tmovl %0,%%ds\n"
 198                 "\tmovl %0,%%es\n"
 199                 "\tmovl %0,%%ss\n"
 200                 "\tmovl %0,%%fs\n"
 201                 "\tmovl %0,%%gs\n"
 202                 : : "a" (__KERNEL_DS) : "memory"
 203                 );
 204 }
 205
 206 #ifdef CONFIG_KEXEC_FILE
 207 /* Update purgatory as needed after various image segments have been prepared */
 208 static int arch_update_purgatory(struct kimage *image)
 209 {
 210         int ret = 0;
 211
 212         if (!image->file_mode)
 213                 return 0;
 214
 215         /* Setup copying of backup region */
 216         if (image->type == KEXEC_TYPE_CRASH) {
 217                 ret = kexec_purgatory_get_set_symbol(image,
 218                                 "purgatory_backup_dest",
 219                                 &image->arch.backup_load_addr,
 220                                 sizeof(image->arch.backup_load_addr), 0);
 221                 if (ret)
 222                         return ret;
 223
 224                 ret = kexec_purgatory_get_set_symbol(image,
 225                                 "purgatory_backup_src",
 226                                 &image->arch.backup_src_start,
 227                                 sizeof(image->arch.backup_src_start), 0);
 228                 if (ret)
 229                         return ret;
 230
 231                 ret = kexec_purgatory_get_set_symbol(image,
 232                                 "purgatory_backup_sz",
 233                                 &image->arch.backup_src_sz,
 234                                 sizeof(image->arch.backup_src_sz), 0);
 235                 if (ret)
 236                         return ret;
 237         }
 238
 239         return ret;
 240 }
 241 #else /* !CONFIG_KEXEC_FILE */
 242 static inline int arch_update_purgatory(struct kimage *image)
 243 {
 244         return 0;
 245 }
 246 #endif /* CONFIG_KEXEC_FILE */
 247
 248 int machine_kexec_prepare(struct kimage *image)
 249 {
 250         unsigned long start_pgtable;
 251         int result;
 252
 253         /* Calculate the offsets */
 254         start_pgtable = page_to_pfn(image->control_code_page) << PAGE_SHIFT;
 255
 256         /* Setup the identity mapped 64bit page table */
 257         result = init_pgtable(image, start_pgtable);
 258         if (result)
 259                 return result;
 260
 261         /* update purgatory as needed */
 262         result = arch_update_purgatory(image);
 263         if (result)
 264                 return result;
 265
 266         return 0;
 267 }
 268
 269 void machine_kexec_cleanup(struct kimage *image)
 270 {
 271         free_transition_pgtable(image);
 272 }
 273
 274 /*
 275  * Do not allocate memory (or fail in any way) in machine_kexec().
 276  * We are past the point of no return, committed to rebooting now.
 277  */
 278 void machine_kexec(struct kimage *image)
 279 {
 280         unsigned long page_list[PAGES_NR];
 281         void *control_page;
 282         int save_ftrace_enabled;
 283
 284 #ifdef CONFIG_KEXEC_JUMP
 285         if (image->preserve_context)
 286                 save_processor_state();
 287 #endif
 288
 289         save_ftrace_enabled = __ftrace_enabled_save();
 290
 291         /* Interrupts aren't acceptable while we reboot */
 292         local_irq_disable();
 293         hw_breakpoint_disable();
 294
 295         if (image->preserve_context) {
 296 #ifdef CONFIG_X86_IO_APIC
 297                 /*
 298                  * We need to put APICs in legacy mode so that we can
 299                  * get timer interrupts in second kernel. kexec/kdump
 300                  * paths already have calls to restore_boot_irq_mode()
 301                  * in one form or other. kexec jump path also need one.
 302                  */
 303                 clear_IO_APIC();
 304                 restore_boot_irq_mode();
 305 #endif
 306         }
 307
 308         control_page = page_address(image->control_code_page) + PAGE_SIZE;
 309         memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
 310
 311         page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
 312         page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
 313         page_list[PA_TABLE_PAGE] =
 314           (unsigned long)__pa(page_address(image->control_code_page));
 315
 316         if (image->type == KEXEC_TYPE_DEFAULT)
 317                 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page)
 318                                                 << PAGE_SHIFT);
 319
 320         /*
 321          * The segment registers are funny things, they have both a
 322          * visible and an invisible part.  Whenever the visible part is
 323          * set to a specific selector, the invisible part is loaded
 324          * with from a table in memory.  At no other time is the
 325          * descriptor table in memory accessed.
 326          *
 327          * I take advantage of this here by force loading the
 328          * segments, before I zap the gdt with an invalid value.
 329          */
 330         load_segments();
 331         /*
 332          * The gdt & idt are now invalid.
 333          * If you want to load them you must set up your own idt & gdt.
 334          */
 335         set_gdt(phys_to_virt(0), 0);
 336         set_idt(phys_to_virt(0), 0);
 337
 338         /* now call it */
 339         image->start = relocate_kernel((unsigned long)image->head,
 340                                        (unsigned long)page_list,
 341                                        image->start,
 342                                        image->preserve_context,
 343                                        sme_active());
 344
 345 #ifdef CONFIG_KEXEC_JUMP
 346         if (image->preserve_context)
 347                 restore_processor_state();
 348 #endif
 349
 350         __ftrace_enabled_restore(save_ftrace_enabled);
 351 }
 352
 353 void arch_crash_save_vmcoreinfo(void)
 354 {
 355         u64 sme_mask = sme_me_mask;
 356
 357         VMCOREINFO_NUMBER(phys_base);
 358         VMCOREINFO_SYMBOL(init_top_pgt);
 359         vmcoreinfo_append_str("NUMBER(pgtable_l5_enabled)=%d\n",
 360                         pgtable_l5_enabled());
 361
 362 #ifdef CONFIG_NUMA
 363         VMCOREINFO_SYMBOL(node_data);
 364         VMCOREINFO_LENGTH(node_data, MAX_NUMNODES);
 365 #endif
 366         vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
 367                               kaslr_offset());
 368         VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE);
 369         VMCOREINFO_NUMBER(sme_mask);
 370 }
 371
 372 /* arch-dependent functionality related to kexec file-based syscall */
 373
 374 #ifdef CONFIG_KEXEC_FILE
 375 void *arch_kexec_kernel_image_load(struct kimage *image)
 376 {
 377         vfree(image->arch.elf_headers);
 378         image->arch.elf_headers = NULL;
 379
 380         if (!image->fops || !image->fops->load)
 381                 return ERR_PTR(-ENOEXEC);
 382
 383         return image->fops->load(image, image->kernel_buf,
 384                                  image->kernel_buf_len, image->initrd_buf,
 385                                  image->initrd_buf_len, image->cmdline_buf,
 386                                  image->cmdline_buf_len);
 387 }
 388
 389 /*
 390  * Apply purgatory relocations.
 391  *
 392  * @pi:         Purgatory to be relocated.
 393  * @section:    Section relocations applying to.
 394  * @relsec:     Section containing RELAs.
 395  * @symtabsec:  Corresponding symtab.
 396  *
 397  * TODO: Some of the code belongs to generic code. Move that in kexec.c.
 398  */
 399 int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
 400                                      Elf_Shdr *section, const Elf_Shdr *relsec,
 401                                      const Elf_Shdr *symtabsec)
 402 {
 403         unsigned int i;
 404         Elf64_Rela *rel;
 405         Elf64_Sym *sym;
 406         void *location;
 407         unsigned long address, sec_base, value;
 408         const char *strtab, *name, *shstrtab;
 409         const Elf_Shdr *sechdrs;
 410
 411         /* String & section header string table */
 412         sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
 413         strtab = (char *)pi->ehdr + sechdrs[symtabsec->sh_link].sh_offset;
 414         shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
 415
 416         rel = (void *)pi->ehdr + relsec->sh_offset;
 417
 418         pr_debug("Applying relocate section %s to %u\n",
 419                  shstrtab + relsec->sh_name, relsec->sh_info);
 420
 421         for (i = 0; i < relsec->sh_size / sizeof(*rel); i++) {
 422
 423                 /*
 424                  * rel[i].r_offset contains byte offset from beginning
 425                  * of section to the storage unit affected.
 426                  *
 427                  * This is location to update. This is temporary buffer
 428                  * where section is currently loaded. This will finally be
 429                  * loaded to a different address later, pointed to by
 430                  * ->sh_addr. kexec takes care of moving it
 431                  *  (kexec_load_segment()).
 432                  */
 433                 location = pi->purgatory_buf;
 434                 location += section->sh_offset;
 435                 location += rel[i].r_offset;
 436
 437                 /* Final address of the location */
 438                 address = section->sh_addr + rel[i].r_offset;
 439
 440                 /*
 441                  * rel[i].r_info contains information about symbol table index
 442                  * w.r.t which relocation must be made and type of relocation
 443                  * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
 444                  * these respectively.
 445                  */
 446                 sym = (void *)pi->ehdr + symtabsec->sh_offset;
 447                 sym += ELF64_R_SYM(rel[i].r_info);
 448
 449                 if (sym->st_name)
 450                         name = strtab + sym->st_name;
 451                 else
 452                         name = shstrtab + sechdrs[sym->st_shndx].sh_name;
 453
 454                 pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
 455                          name, sym->st_info, sym->st_shndx, sym->st_value,
 456                          sym->st_size);
 457
 458                 if (sym->st_shndx == SHN_UNDEF) {
 459                         pr_err("Undefined symbol: %s\n", name);
 460                         return -ENOEXEC;
 461                 }
 462
 463                 if (sym->st_shndx == SHN_COMMON) {
 464                         pr_err("symbol '%s' in common section\n", name);
 465                         return -ENOEXEC;
 466                 }
 467
 468                 if (sym->st_shndx == SHN_ABS)
 469                         sec_base = 0;
 470                 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
 471                         pr_err("Invalid section %d for symbol %s\n",
 472                                sym->st_shndx, name);
 473                         return -ENOEXEC;
 474                 } else
 475                         sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
 476
 477                 value = sym->st_value;
 478                 value += sec_base;
 479                 value += rel[i].r_addend;
 480
 481                 switch (ELF64_R_TYPE(rel[i].r_info)) {
 482                 case R_X86_64_NONE:
 483                         break;
 484                 case R_X86_64_64:
 485                         *(u64 *)location = value;
 486                         break;
 487                 case R_X86_64_32:
 488                         *(u32 *)location = value;
 489                         if (value != *(u32 *)location)
 490                                 goto overflow;
 491                         break;
 492                 case R_X86_64_32S:
 493                         *(s32 *)location = value;
 494                         if ((s64)value != *(s32 *)location)
 495                                 goto overflow;
 496                         break;
 497                 case R_X86_64_PC32:
 498                 case R_X86_64_PLT32:
 499                         value -= (u64)address;
 500                         *(u32 *)location = value;
 501                         break;
 502                 default:
 503                         pr_err("Unknown rela relocation: %llu\n",
 504                                ELF64_R_TYPE(rel[i].r_info));
 505                         return -ENOEXEC;
 506                 }
 507         }
 508         return 0;
 509
 510 overflow:
 511         pr_err("Overflow in relocation type %d value 0x%lx\n",
 512                (int)ELF64_R_TYPE(rel[i].r_info), value);
 513         return -ENOEXEC;
 514 }
 515 #endif /* CONFIG_KEXEC_FILE */
 516
 517 static int
 518 kexec_mark_range(unsigned long start, unsigned long end, bool protect)
 519 {
 520         struct page *page;
 521         unsigned int nr_pages;
 522
 523         /*
 524          * For physical range: [start, end]. We must skip the unassigned
 525          * crashk resource with zero-valued "end" member.
 526          */
 527         if (!end || start > end)
 528                 return 0;
 529
 530         page = pfn_to_page(start >> PAGE_SHIFT);
 531         nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
 532         if (protect)
 533                 return set_pages_ro(page, nr_pages);
 534         else
 535                 return set_pages_rw(page, nr_pages);
 536 }
 537
 538 static void kexec_mark_crashkres(bool protect)
 539 {
 540         unsigned long control;
 541
 542         kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect);
 543
 544         /* Don't touch the control code page used in crash_kexec().*/
 545         control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page));
 546         /* Control code page is located in the 2nd page. */
 547         kexec_mark_range(crashk_res.start, control + PAGE_SIZE - 1, protect);
 548         control += KEXEC_CONTROL_PAGE_SIZE;
 549         kexec_mark_range(control, crashk_res.end, protect);
 550 }
 551
 552 void arch_kexec_protect_crashkres(void)
 553 {
 554         kexec_mark_crashkres(true);
 555 }
 556
 557 void arch_kexec_unprotect_crashkres(void)
 558 {
 559         kexec_mark_crashkres(false);
 560 }
 561
 562 int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp)
 563 {
 564         /*
 565          * If SME is active we need to be sure that kexec pages are
 566          * not encrypted because when we boot to the new kernel the
 567          * pages won't be accessed encrypted (initially).
 568          */
 569         return set_memory_decrypted((unsigned long)vaddr, pages);
 570 }
 571
 572 void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages)
 573 {
 574         /*
 575          * If SME is active we need to reset the pages back to being
 576          * an encrypted mapping before freeing them.
 577          */
 578         set_memory_encrypted((unsigned long)vaddr, pages);
 579 }