diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index e468476e9e4a..c6512f2ea31e 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -198,11 +198,11 @@ bool sev_es_check_ghcb_fault(unsigned long address) #endif /* - * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented + * SNP_FEATURES_IMPL is the mask of SNP features that are implemented * by the guest kernel. As and when a new feature is implemented in the * guest kernel, a corresponding bit should be added to the mask. */ -#define SNP_FEATURES_PRESENT (MSR_AMD64_SNP_DEBUG_SWAP | \ +#define SNP_FEATURES_IMPL (MSR_AMD64_SNP_DEBUG_SWAP | \ MSR_AMD64_SNP_SECURE_TSC | \ SNP_FEATURE_SECURE_AVIC) @@ -211,7 +211,7 @@ u64 snp_get_unsupported_features(u64 status) if (!(status & MSR_AMD64_SEV_SNP_ENABLED)) return 0; - return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT; + return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_IMPL; } void snp_check_features(void) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 0e6c0940100f..09e605c85de4 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -661,6 +661,8 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages) { __snp_leak_pages(pfn, pages, true); } +void snp_prepare(void); +void snp_shutdown(void); #else static inline bool snp_probe_rmptable_info(void) { return false; } static inline int snp_rmptable_init(void) { return -ENOSYS; } @@ -677,6 +679,8 @@ static inline void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp) static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} static inline void kdump_sev_callback(void) { } static inline void snp_fixup_e820_tables(void) {} +static inline void snp_prepare(void) {} +static inline void snp_shutdown(void) {} #endif #endif diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index a4f3a364fb65..41f76f15caa1 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -117,6 +117,8 @@ static u64 rmp_segment_mask; static u64 rmp_cfg; +static void *rmp_bookkeeping __ro_after_init; + /* Mask to apply to a PFN to get the first PFN of a 2MB page */ #define PFN_PMD_MASK GENMASK_ULL(63, PMD_SHIFT - PAGE_SHIFT) @@ -130,33 +132,23 @@ static unsigned long snp_nr_leaked_pages; #undef pr_fmt #define pr_fmt(fmt) "SEV-SNP: " fmt -static int __mfd_enable(unsigned int cpu) +static void mfd_reconfigure(void *arg) +{ + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return; + + if (arg) + msr_set_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT); + else + msr_clear_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT); +} + +static void snp_enable(void *arg) { u64 val; if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) - return 0; - - rdmsrq(MSR_AMD64_SYSCFG, val); - - val |= MSR_AMD64_SYSCFG_MFDM; - - wrmsrq(MSR_AMD64_SYSCFG, val); - - return 0; -} - -static __init void mfd_enable(void *arg) -{ - __mfd_enable(smp_processor_id()); -} - -static int __snp_enable(unsigned int cpu) -{ - u64 val; - - if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) - return 0; + return; rdmsrq(MSR_AMD64_SYSCFG, val); @@ -164,13 +156,6 @@ static int __snp_enable(unsigned int cpu) val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN; wrmsrq(MSR_AMD64_SYSCFG, val); - - return 0; -} - -static __init void snp_enable(void *arg) -{ - __snp_enable(smp_processor_id()); } static void __init __snp_fixup_e820_tables(u64 pa) @@ -260,21 +245,30 @@ void __init snp_fixup_e820_tables(void) } } -static bool __init clear_rmptable_bookkeeping(void) +static void clear_rmp(void) { - void *bk; + unsigned int i; + u64 val; - bk = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB); - if (!bk) { - pr_err("Failed to map RMP bookkeeping area\n"); - return false; + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return; + + /* Clearing the RMP while SNP is enabled will cause an exception */ + rdmsrq(MSR_AMD64_SYSCFG, val); + if (WARN_ON_ONCE(val & MSR_AMD64_SYSCFG_SNP_EN)) + return; + + memset(rmp_bookkeeping, 0, RMPTABLE_CPU_BOOKKEEPING_SZ); + + for (i = 0; i < rst_max_index; i++) { + struct rmp_segment_desc *desc; + + desc = rmp_segment_table[i]; + if (!desc) + continue; + + memset(desc->rmp_entry, 0, desc->size); } - - memset(bk, 0, RMPTABLE_CPU_BOOKKEEPING_SZ); - - memunmap(bk); - - return true; } static bool __init alloc_rmp_segment_desc(u64 segment_pa, u64 segment_size, u64 pa) @@ -494,12 +488,72 @@ e_free: static bool __init setup_rmptable(void) { if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) { - return setup_segmented_rmptable(); + if (!setup_segmented_rmptable()) + return false; } else { - return setup_contiguous_rmptable(); + if (!setup_contiguous_rmptable()) + return false; } + + rmp_bookkeeping = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB); + if (!rmp_bookkeeping) { + pr_err("Failed to map RMP bookkeeping area\n"); + free_rmp_segment_table(); + + return false; + } + + return true; } +static void clear_hsave_pa(void *arg) +{ + wrmsrq(MSR_VM_HSAVE_PA, 0); +} + +void snp_prepare(void) +{ + u64 val; + + /* + * Check if SEV-SNP is already enabled, this can happen in case of + * kexec boot. + */ + rdmsrq(MSR_AMD64_SYSCFG, val); + if (val & MSR_AMD64_SYSCFG_SNP_EN) + return; + + clear_rmp(); + + cpus_read_lock(); + + /* + * MtrrFixDramModEn is not shared between threads on a core, + * therefore it must be set on all CPUs prior to enabling SNP. + */ + on_each_cpu(mfd_reconfigure, (void *)1, 1); + on_each_cpu(snp_enable, NULL, 1); + + /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ + on_each_cpu(clear_hsave_pa, NULL, 1); + + cpus_read_unlock(); +} +EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp"); + +void snp_shutdown(void) +{ + u64 syscfg; + + rdmsrq(MSR_AMD64_SYSCFG, syscfg); + if (syscfg & MSR_AMD64_SYSCFG_SNP_EN) + return; + + clear_rmp(); + on_each_cpu(mfd_reconfigure, NULL, 1); +} +EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp"); + /* * Do the necessary preparations which are verified by the firmware as * described in the SNP_INIT_EX firmware command description in the SNP @@ -507,9 +561,6 @@ static bool __init setup_rmptable(void) */ int __init snp_rmptable_init(void) { - unsigned int i; - u64 val; - if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP))) return -ENOSYS; @@ -519,42 +570,6 @@ int __init snp_rmptable_init(void) if (!setup_rmptable()) return -ENOSYS; - /* - * Check if SEV-SNP is already enabled, this can happen in case of - * kexec boot. - */ - rdmsrq(MSR_AMD64_SYSCFG, val); - if (val & MSR_AMD64_SYSCFG_SNP_EN) - goto skip_enable; - - /* Zero out the RMP bookkeeping area */ - if (!clear_rmptable_bookkeeping()) { - free_rmp_segment_table(); - return -ENOSYS; - } - - /* Zero out the RMP entries */ - for (i = 0; i < rst_max_index; i++) { - struct rmp_segment_desc *desc; - - desc = rmp_segment_table[i]; - if (!desc) - continue; - - memset(desc->rmp_entry, 0, desc->size); - } - - /* Flush the caches to ensure that data is written before SNP is enabled. */ - wbinvd_on_all_cpus(); - - /* MtrrFixDramModEn must be enabled on all the CPUs prior to enabling SNP. */ - on_each_cpu(mfd_enable, NULL, 1); - - on_each_cpu(snp_enable, NULL, 1); - -skip_enable: - cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/rmptable_init:online", __snp_enable, NULL); - /* * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic * notifier is invoked to do SNP IOMMU shutdown before kdump. diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index aebf4dad545e..939fa8aa155c 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1076,11 +1076,6 @@ static inline int __sev_do_init_locked(int *psp_ret) return __sev_init_locked(psp_ret); } -static void snp_set_hsave_pa(void *arg) -{ - wrmsrq(MSR_VM_HSAVE_PA, 0); -} - /* Hypervisor Fixed pages API interface */ static void snp_hv_fixed_pages_state_update(struct sev_device *sev, enum snp_hv_fixed_pages_state page_state) @@ -1224,7 +1219,7 @@ static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range static void snp_leak_hv_fixed_pages(void) { - struct snp_hv_fixed_pages_entry *entry; + struct snp_hv_fixed_pages_entry *entry, *nentry; /* List is protected by sev_cmd_mutex */ lockdep_assert_held(&sev_cmd_mutex); @@ -1232,10 +1227,16 @@ static void snp_leak_hv_fixed_pages(void) if (list_empty(&snp_hv_fixed_pages)) return; - list_for_each_entry(entry, &snp_hv_fixed_pages, list) - if (entry->page_state == HV_FIXED) + list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) { + if (entry->free && entry->page_state != HV_FIXED) + __free_pages(entry->page, entry->order); + else __snp_leak_pages(page_to_pfn(entry->page), 1 << entry->order, false); + + list_del(&entry->list); + kfree(entry); + } } bool sev_is_snp_ciphertext_hiding_supported(void) @@ -1373,8 +1374,7 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) return -EOPNOTSUPP; } - /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ - on_each_cpu(snp_set_hsave_pa, NULL, 1); + snp_prepare(); /* * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list @@ -2045,6 +2045,8 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) memset(&data, 0, sizeof(data)); data.len = sizeof(data); data.iommu_snp_shutdown = 1; + if (sev->snp_feat_info_0.ecx & SNP_X86_SHUTDOWN_SUPPORTED) + data.x86_snp_shutdown = 1; /* * If invoked during panic handling, local interrupts are disabled @@ -2078,23 +2080,29 @@ static int __sev_snp_shutdown_locked(int *error, bool panic) return ret; } - /* - * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP - * enforcement by the IOMMU and also transitions all pages - * associated with the IOMMU to the Reclaim state. - * Firmware was transitioning the IOMMU pages to Hypervisor state - * before version 1.53. But, accounting for the number of assigned - * 4kB pages in a 2M page was done incorrectly by not transitioning - * to the Reclaim state. This resulted in RMP #PF when later accessing - * the 2M page containing those pages during kexec boot. Hence, the - * firmware now transitions these pages to Reclaim state and hypervisor - * needs to transition these pages to shared state. SNP Firmware - * version 1.53 and above are needed for kexec boot. - */ - ret = amd_iommu_snp_disable(); - if (ret) { - dev_err(sev->dev, "SNP IOMMU shutdown failed\n"); - return ret; + if (data.x86_snp_shutdown) { + if (!panic) + snp_shutdown(); + snp_hv_fixed_pages_state_update(sev, ALLOCATED); + } else { + /* + * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP + * enforcement by the IOMMU and also transitions all pages + * associated with the IOMMU to the Reclaim state. + * Firmware was transitioning the IOMMU pages to Hypervisor state + * before version 1.53. But, accounting for the number of assigned + * 4kB pages in a 2M page was done incorrectly by not transitioning + * to the Reclaim state. This resulted in RMP #PF when later accessing + * the 2M page containing those pages during kexec boot. Hence, the + * firmware now transitions these pages to Reclaim state and hypervisor + * needs to transition these pages to shared state. SNP Firmware + * version 1.53 and above are needed for kexec boot. + */ + ret = amd_iommu_snp_disable(); + if (ret) { + dev_err(sev->dev, "SNP IOMMU shutdown failed\n"); + return ret; + } } snp_leak_hv_fixed_pages(); diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index 69ffa4b4d1fa..d5099a2baca5 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -829,12 +829,14 @@ struct sev_data_range_list { * * @len: length of the command buffer read by the PSP * @iommu_snp_shutdown: Disable enforcement of SNP in the IOMMU + * @x86_snp_shutdown: Disable SNP on all cores * @rsvd1: reserved */ struct sev_data_snp_shutdown_ex { u32 len; u32 iommu_snp_shutdown:1; - u32 rsvd1:31; + u32 x86_snp_shutdown:1; + u32 rsvd1:30; } __packed; /** @@ -891,6 +893,7 @@ struct snp_feature_info { } __packed; /* Feature bits in ECX */ +#define SNP_X86_SHUTDOWN_SUPPORTED BIT(1) #define SNP_RAPL_DISABLE_SUPPORTED BIT(2) #define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3) #define SNP_AES_256_XTS_POLICY_SUPPORTED BIT(4)