1 // SPDX-License-Identifier: GPL-2.0
3 * Common Ultravisor functions and initialization
5 * Copyright IBM Corp. 2019, 2020
7 #define KMSG_COMPONENT "prot_virt"
8 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10 #include <linux/kernel.h>
11 #include <linux/types.h>
12 #include <linux/sizes.h>
13 #include <linux/bitmap.h>
14 #include <linux/memblock.h>
15 #include <linux/pagemap.h>
16 #include <linux/swap.h>
17 #include <asm/facility.h>
18 #include <asm/sections.h>
21 /* the bootdata_preserved fields come from ones in arch/s390/boot/uv.c */
22 #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
23 int __bootdata_preserved(prot_virt_guest);
24 EXPORT_SYMBOL(prot_virt_guest);
28 * uv_info contains both host and guest information but it's currently only
29 * expected to be used within modules if it's the KVM module or for
30 * any PV guest module.
32 * The kernel itself will write these values once in uv_query_info()
33 * and then make some of them readable via a sysfs interface.
35 struct uv_info __bootdata_preserved(uv_info);
36 EXPORT_SYMBOL(uv_info);
38 #if IS_ENABLED(CONFIG_KVM)
39 int __bootdata_preserved(prot_virt_host);
40 EXPORT_SYMBOL(prot_virt_host);
42 static int __init uv_init(phys_addr_t stor_base, unsigned long stor_len)
44 struct uv_cb_init uvcb = {
45 .header.cmd = UVC_CMD_INIT_UV,
46 .header.len = sizeof(uvcb),
47 .stor_origin = stor_base,
51 if (uv_call(0, (uint64_t)&uvcb)) {
52 pr_err("Ultravisor init failed with rc: 0x%x rrc: 0%x\n",
53 uvcb.header.rc, uvcb.header.rrc);
59 void __init setup_uv(void)
63 if (!is_prot_virt_host())
66 uv_stor_base = memblock_alloc_try_nid(
67 uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
68 MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
70 pr_warn("Failed to reserve %lu bytes for ultravisor base storage\n",
71 uv_info.uv_base_stor_len);
75 if (uv_init(__pa(uv_stor_base), uv_info.uv_base_stor_len)) {
76 memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
80 pr_info("Reserving %luMB as ultravisor base storage\n",
81 uv_info.uv_base_stor_len >> 20);
84 pr_info("Disabling support for protected virtualization");
89 * Requests the Ultravisor to pin the page in the shared state. This will
90 * cause an intercept when the guest attempts to unshare the pinned page.
92 int uv_pin_shared(unsigned long paddr)
94 struct uv_cb_cfs uvcb = {
95 .header.cmd = UVC_CMD_PIN_PAGE_SHARED,
96 .header.len = sizeof(uvcb),
100 if (uv_call(0, (u64)&uvcb))
104 EXPORT_SYMBOL_GPL(uv_pin_shared);
107 * Requests the Ultravisor to destroy a guest page and make it
108 * accessible to the host. The destroy clears the page instead of
111 * @paddr: Absolute host address of page to be destroyed
113 static int uv_destroy_page(unsigned long paddr)
115 struct uv_cb_cfs uvcb = {
116 .header.cmd = UVC_CMD_DESTR_SEC_STOR,
117 .header.len = sizeof(uvcb),
121 if (uv_call(0, (u64)&uvcb)) {
123 * Older firmware uses 107/d as an indication of a non secure
124 * page. Let us emulate the newer variant (no-op).
126 if (uvcb.header.rc == 0x107 && uvcb.header.rrc == 0xd)
134 * The caller must already hold a reference to the page
136 int uv_destroy_owned_page(unsigned long paddr)
138 struct page *page = phys_to_page(paddr);
142 rc = uv_destroy_page(paddr);
144 clear_bit(PG_arch_1, &page->flags);
150 * Requests the Ultravisor to encrypt a guest page and make it
151 * accessible to the host for paging (export).
153 * @paddr: Absolute host address of page to be exported
155 int uv_convert_from_secure(unsigned long paddr)
157 struct uv_cb_cfs uvcb = {
158 .header.cmd = UVC_CMD_CONV_FROM_SEC_STOR,
159 .header.len = sizeof(uvcb),
163 if (uv_call(0, (u64)&uvcb))
169 * The caller must already hold a reference to the page
171 int uv_convert_owned_from_secure(unsigned long paddr)
173 struct page *page = phys_to_page(paddr);
177 rc = uv_convert_from_secure(paddr);
179 clear_bit(PG_arch_1, &page->flags);
185 * Calculate the expected ref_count for a folio that would otherwise have no
186 * further pins. This was cribbed from similar functions in other places in
187 * the kernel, but with some slight modifications. We know that a secure
188 * folio can not be a large folio, for example.
190 static int expected_folio_refs(struct folio *folio)
194 res = folio_mapcount(folio);
195 if (folio_test_swapcache(folio)) {
197 } else if (folio_mapping(folio)) {
205 static int make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb)
207 int expected, cc = 0;
209 if (folio_test_writeback(folio))
211 expected = expected_folio_refs(folio);
212 if (!folio_ref_freeze(folio, expected))
214 set_bit(PG_arch_1, &folio->flags);
216 * If the UVC does not succeed or fail immediately, we don't want to
217 * loop for long, or we might get stall notifications.
218 * On the other hand, this is a complex scenario and we are holding a lot of
219 * locks, so we can't easily sleep and reschedule. We try only once,
220 * and if the UVC returned busy or partial completion, we return
221 * -EAGAIN and we let the callers deal with it.
223 cc = __uv_call(0, (u64)uvcb);
224 folio_ref_unfreeze(folio, expected);
226 * Return -ENXIO if the folio was not mapped, -EINVAL for other errors.
227 * If busy or partially completed, return -EAGAIN.
231 else if (cc == UVC_CC_BUSY || cc == UVC_CC_PARTIAL)
233 return uvcb->rc == 0x10a ? -ENXIO : -EINVAL;
237 * should_export_before_import - Determine whether an export is needed
238 * before an import-like operation
239 * @uvcb: the Ultravisor control block of the UVC to be performed
240 * @mm: the mm of the process
242 * Returns whether an export is needed before every import-like operation.
243 * This is needed for shared pages, which don't trigger a secure storage
244 * exception when accessed from a different guest.
246 * Although considered as one, the Unpin Page UVC is not an actual import,
247 * so it is not affected.
249 * No export is needed also when there is only one protected VM, because the
250 * page cannot belong to the wrong VM in that case (there is no "other VM"
253 * Return: true if an export is needed before every import, otherwise false.
255 static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm)
258 * The misc feature indicates, among other things, that importing a
259 * shared page from a different protected VM will automatically also
260 * transfer its ownership.
262 if (uv_has_feature(BIT_UV_FEAT_MISC))
264 if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED)
266 return atomic_read(&mm->context.protected_count) > 1;
270 * Requests the Ultravisor to make a page accessible to a guest.
271 * If it's brought in the first time, it will be cleared. If
272 * it has been exported before, it will be decrypted and integrity
275 int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb)
277 struct vm_area_struct *vma;
278 bool local_drain = false;
287 mmap_read_lock(gmap->mm);
289 uaddr = __gmap_translate(gmap, gaddr);
290 if (IS_ERR_VALUE(uaddr))
292 vma = vma_lookup(gmap->mm, uaddr);
296 * Secure pages cannot be huge and userspace should not combine both.
297 * In case userspace does it anyway this will result in an -EFAULT for
298 * the unpack. The guest is thus never reaching secure mode. If
299 * userspace is playing dirty tricky with mapping huge pages later
300 * on this will result in a segmentation fault.
302 if (is_vm_hugetlb_page(vma))
306 ptep = get_locked_pte(gmap->mm, uaddr, &ptelock);
309 if (pte_present(*ptep) && !(pte_val(*ptep) & _PAGE_INVALID) && pte_write(*ptep)) {
310 folio = page_folio(pte_page(*ptep));
312 if (folio_test_large(folio))
315 if (folio_trylock(folio)) {
316 if (should_export_before_import(uvcb, gmap->mm))
317 uv_convert_from_secure(PFN_PHYS(folio_pfn(folio)));
318 rc = make_folio_secure(folio, uvcb);
323 pte_unmap_unlock(ptep, ptelock);
325 mmap_read_unlock(gmap->mm);
329 * If we are here because the UVC returned busy or partial
330 * completion, this is just a useless check, but it is safe.
332 folio_wait_writeback(folio);
333 } else if (rc == -EBUSY) {
335 * If we have tried a local drain and the folio refcount
336 * still does not match our expected safe value, try with a
337 * system wide drain. This is needed if the pagevecs holding
338 * the page are on a different CPU.
342 /* We give up here, and let the caller try again */
346 * We are here if the folio refcount does not match the
347 * expected safe value. The main culprits are usually
348 * pagevecs. With lru_add_drain() we drain the pagevecs
349 * on the local CPU so that hopefully the refcount will
350 * reach the expected safe value.
354 /* And now we try again immediately after draining */
356 } else if (rc == -ENXIO) {
357 if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE))
363 EXPORT_SYMBOL_GPL(gmap_make_secure);
365 int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr)
367 struct uv_cb_cts uvcb = {
368 .header.cmd = UVC_CMD_CONV_TO_SEC_STOR,
369 .header.len = sizeof(uvcb),
370 .guest_handle = gmap->guest_handle,
374 return gmap_make_secure(gmap, gaddr, &uvcb);
376 EXPORT_SYMBOL_GPL(gmap_convert_to_secure);
379 * gmap_destroy_page - Destroy a guest page.
380 * @gmap: the gmap of the guest
381 * @gaddr: the guest address to destroy
383 * An attempt will be made to destroy the given guest page. If the attempt
384 * fails, an attempt is made to export the page. If both attempts fail, an
385 * appropriate error is returned.
387 int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr)
389 struct vm_area_struct *vma;
395 mmap_read_lock(gmap->mm);
397 uaddr = __gmap_translate(gmap, gaddr);
398 if (IS_ERR_VALUE(uaddr))
400 vma = vma_lookup(gmap->mm, uaddr);
404 * Huge pages should not be able to become secure
406 if (is_vm_hugetlb_page(vma))
410 /* we take an extra reference here */
411 page = follow_page(vma, uaddr, FOLL_WRITE | FOLL_GET);
412 if (IS_ERR_OR_NULL(page))
414 rc = uv_destroy_owned_page(page_to_phys(page));
416 * Fault handlers can race; it is possible that two CPUs will fault
417 * on the same secure page. One CPU can destroy the page, reboot,
418 * re-enter secure mode and import it, while the second CPU was
419 * stuck at the beginning of the handler. At some point the second
420 * CPU will be able to progress, and it will not be able to destroy
421 * the page. In that case we do not want to terminate the process,
422 * we instead try to export the page.
425 rc = uv_convert_owned_from_secure(page_to_phys(page));
428 mmap_read_unlock(gmap->mm);
431 EXPORT_SYMBOL_GPL(gmap_destroy_page);
434 * To be called with the page locked or with an extra reference! This will
435 * prevent gmap_make_secure from touching the page concurrently. Having 2
436 * parallel make_page_accessible is fine, as the UV calls will become a
437 * no-op if the page is already exported.
439 int arch_make_page_accessible(struct page *page)
443 /* Hugepage cannot be protected, so nothing to do */
448 * PG_arch_1 is used in 3 places:
449 * 1. for kernel page tables during early boot
450 * 2. for storage keys of huge pages and KVM
451 * 3. As an indication that this page might be secure. This can
452 * overindicate, e.g. we set the bit before calling
454 * As secure pages are never huge, all 3 variants can co-exists.
456 if (!test_bit(PG_arch_1, &page->flags))
459 rc = uv_pin_shared(page_to_phys(page));
461 clear_bit(PG_arch_1, &page->flags);
465 rc = uv_convert_from_secure(page_to_phys(page));
467 clear_bit(PG_arch_1, &page->flags);
473 EXPORT_SYMBOL_GPL(arch_make_page_accessible);
477 #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM)
478 static ssize_t uv_query_facilities(struct kobject *kobj,
479 struct kobj_attribute *attr, char *buf)
481 return sysfs_emit(buf, "%lx\n%lx\n%lx\n%lx\n",
482 uv_info.inst_calls_list[0],
483 uv_info.inst_calls_list[1],
484 uv_info.inst_calls_list[2],
485 uv_info.inst_calls_list[3]);
488 static struct kobj_attribute uv_query_facilities_attr =
489 __ATTR(facilities, 0444, uv_query_facilities, NULL);
491 static ssize_t uv_query_supp_se_hdr_ver(struct kobject *kobj,
492 struct kobj_attribute *attr, char *buf)
494 return sysfs_emit(buf, "%lx\n", uv_info.supp_se_hdr_ver);
497 static struct kobj_attribute uv_query_supp_se_hdr_ver_attr =
498 __ATTR(supp_se_hdr_ver, 0444, uv_query_supp_se_hdr_ver, NULL);
500 static ssize_t uv_query_supp_se_hdr_pcf(struct kobject *kobj,
501 struct kobj_attribute *attr, char *buf)
503 return sysfs_emit(buf, "%lx\n", uv_info.supp_se_hdr_pcf);
506 static struct kobj_attribute uv_query_supp_se_hdr_pcf_attr =
507 __ATTR(supp_se_hdr_pcf, 0444, uv_query_supp_se_hdr_pcf, NULL);
509 static ssize_t uv_query_dump_cpu_len(struct kobject *kobj,
510 struct kobj_attribute *attr, char *buf)
512 return sysfs_emit(buf, "%lx\n", uv_info.guest_cpu_stor_len);
515 static struct kobj_attribute uv_query_dump_cpu_len_attr =
516 __ATTR(uv_query_dump_cpu_len, 0444, uv_query_dump_cpu_len, NULL);
518 static ssize_t uv_query_dump_storage_state_len(struct kobject *kobj,
519 struct kobj_attribute *attr, char *buf)
521 return sysfs_emit(buf, "%lx\n", uv_info.conf_dump_storage_state_len);
524 static struct kobj_attribute uv_query_dump_storage_state_len_attr =
525 __ATTR(dump_storage_state_len, 0444, uv_query_dump_storage_state_len, NULL);
527 static ssize_t uv_query_dump_finalize_len(struct kobject *kobj,
528 struct kobj_attribute *attr, char *buf)
530 return sysfs_emit(buf, "%lx\n", uv_info.conf_dump_finalize_len);
533 static struct kobj_attribute uv_query_dump_finalize_len_attr =
534 __ATTR(dump_finalize_len, 0444, uv_query_dump_finalize_len, NULL);
536 static ssize_t uv_query_feature_indications(struct kobject *kobj,
537 struct kobj_attribute *attr, char *buf)
539 return sysfs_emit(buf, "%lx\n", uv_info.uv_feature_indications);
542 static struct kobj_attribute uv_query_feature_indications_attr =
543 __ATTR(feature_indications, 0444, uv_query_feature_indications, NULL);
545 static ssize_t uv_query_max_guest_cpus(struct kobject *kobj,
546 struct kobj_attribute *attr, char *buf)
548 return sysfs_emit(buf, "%d\n", uv_info.max_guest_cpu_id + 1);
551 static struct kobj_attribute uv_query_max_guest_cpus_attr =
552 __ATTR(max_cpus, 0444, uv_query_max_guest_cpus, NULL);
554 static ssize_t uv_query_max_guest_vms(struct kobject *kobj,
555 struct kobj_attribute *attr, char *buf)
557 return sysfs_emit(buf, "%d\n", uv_info.max_num_sec_conf);
560 static struct kobj_attribute uv_query_max_guest_vms_attr =
561 __ATTR(max_guests, 0444, uv_query_max_guest_vms, NULL);
563 static ssize_t uv_query_max_guest_addr(struct kobject *kobj,
564 struct kobj_attribute *attr, char *buf)
566 return sysfs_emit(buf, "%lx\n", uv_info.max_sec_stor_addr);
569 static struct kobj_attribute uv_query_max_guest_addr_attr =
570 __ATTR(max_address, 0444, uv_query_max_guest_addr, NULL);
572 static ssize_t uv_query_supp_att_req_hdr_ver(struct kobject *kobj,
573 struct kobj_attribute *attr, char *buf)
575 return sysfs_emit(buf, "%lx\n", uv_info.supp_att_req_hdr_ver);
578 static struct kobj_attribute uv_query_supp_att_req_hdr_ver_attr =
579 __ATTR(supp_att_req_hdr_ver, 0444, uv_query_supp_att_req_hdr_ver, NULL);
581 static ssize_t uv_query_supp_att_pflags(struct kobject *kobj,
582 struct kobj_attribute *attr, char *buf)
584 return sysfs_emit(buf, "%lx\n", uv_info.supp_att_pflags);
587 static struct kobj_attribute uv_query_supp_att_pflags_attr =
588 __ATTR(supp_att_pflags, 0444, uv_query_supp_att_pflags, NULL);
590 static ssize_t uv_query_supp_add_secret_req_ver(struct kobject *kobj,
591 struct kobj_attribute *attr, char *buf)
593 return sysfs_emit(buf, "%lx\n", uv_info.supp_add_secret_req_ver);
596 static struct kobj_attribute uv_query_supp_add_secret_req_ver_attr =
597 __ATTR(supp_add_secret_req_ver, 0444, uv_query_supp_add_secret_req_ver, NULL);
599 static ssize_t uv_query_supp_add_secret_pcf(struct kobject *kobj,
600 struct kobj_attribute *attr, char *buf)
602 return sysfs_emit(buf, "%lx\n", uv_info.supp_add_secret_pcf);
605 static struct kobj_attribute uv_query_supp_add_secret_pcf_attr =
606 __ATTR(supp_add_secret_pcf, 0444, uv_query_supp_add_secret_pcf, NULL);
608 static ssize_t uv_query_supp_secret_types(struct kobject *kobj,
609 struct kobj_attribute *attr, char *buf)
611 return sysfs_emit(buf, "%lx\n", uv_info.supp_secret_types);
614 static struct kobj_attribute uv_query_supp_secret_types_attr =
615 __ATTR(supp_secret_types, 0444, uv_query_supp_secret_types, NULL);
617 static ssize_t uv_query_max_secrets(struct kobject *kobj,
618 struct kobj_attribute *attr, char *buf)
620 return sysfs_emit(buf, "%d\n", uv_info.max_secrets);
623 static struct kobj_attribute uv_query_max_secrets_attr =
624 __ATTR(max_secrets, 0444, uv_query_max_secrets, NULL);
626 static struct attribute *uv_query_attrs[] = {
627 &uv_query_facilities_attr.attr,
628 &uv_query_feature_indications_attr.attr,
629 &uv_query_max_guest_cpus_attr.attr,
630 &uv_query_max_guest_vms_attr.attr,
631 &uv_query_max_guest_addr_attr.attr,
632 &uv_query_supp_se_hdr_ver_attr.attr,
633 &uv_query_supp_se_hdr_pcf_attr.attr,
634 &uv_query_dump_storage_state_len_attr.attr,
635 &uv_query_dump_finalize_len_attr.attr,
636 &uv_query_dump_cpu_len_attr.attr,
637 &uv_query_supp_att_req_hdr_ver_attr.attr,
638 &uv_query_supp_att_pflags_attr.attr,
639 &uv_query_supp_add_secret_req_ver_attr.attr,
640 &uv_query_supp_add_secret_pcf_attr.attr,
641 &uv_query_supp_secret_types_attr.attr,
642 &uv_query_max_secrets_attr.attr,
646 static struct attribute_group uv_query_attr_group = {
647 .attrs = uv_query_attrs,
650 static ssize_t uv_is_prot_virt_guest(struct kobject *kobj,
651 struct kobj_attribute *attr, char *buf)
655 #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
656 val = prot_virt_guest;
658 return sysfs_emit(buf, "%d\n", val);
661 static ssize_t uv_is_prot_virt_host(struct kobject *kobj,
662 struct kobj_attribute *attr, char *buf)
666 #if IS_ENABLED(CONFIG_KVM)
667 val = prot_virt_host;
670 return sysfs_emit(buf, "%d\n", val);
673 static struct kobj_attribute uv_prot_virt_guest =
674 __ATTR(prot_virt_guest, 0444, uv_is_prot_virt_guest, NULL);
676 static struct kobj_attribute uv_prot_virt_host =
677 __ATTR(prot_virt_host, 0444, uv_is_prot_virt_host, NULL);
679 static const struct attribute *uv_prot_virt_attrs[] = {
680 &uv_prot_virt_guest.attr,
681 &uv_prot_virt_host.attr,
685 static struct kset *uv_query_kset;
686 static struct kobject *uv_kobj;
688 static int __init uv_info_init(void)
692 if (!test_facility(158))
695 uv_kobj = kobject_create_and_add("uv", firmware_kobj);
699 rc = sysfs_create_files(uv_kobj, uv_prot_virt_attrs);
703 uv_query_kset = kset_create_and_add("query", NULL, uv_kobj);
704 if (!uv_query_kset) {
709 rc = sysfs_create_group(&uv_query_kset->kobj, &uv_query_attr_group);
713 kset_unregister(uv_query_kset);
715 sysfs_remove_files(uv_kobj, uv_prot_virt_attrs);
717 kobject_del(uv_kobj);
718 kobject_put(uv_kobj);
721 device_initcall(uv_info_init);