commit 758d9fd121c59729955b3bf5c06f953e54ae67d9 Author: m00nbsd <42475391+m00nbsd@users.noreply.github.com> Date: Mon Jul 27 20:53:10 2026 -0400 nvmm_x86_svm: implement a CR0 quirk Some guests have issues where their UEFI bootloader rides with CR0.CD=1 which is extremely slow under SVM. Implement a quirk to force CR0.CD=0 in the guest, akin to KVM's KVM_X86_QUIRK_CD_NW_CLEARED. If the guest attempts to execute an SMSW, or an LMSW when the quirk is active, we abandon the quirk, let the guest execute SMSW/LMSW natively, and we passively reinstate the quirk in the next VCPU iteration. This is to avoid doing heavy decoding and emulation of SMSW/LMSW. diff --git a/src/driver/x86/nvmm_x86_svm.c b/src/driver/x86/nvmm_x86_svm.c index edc97df..bbb2485 100644 --- a/src/driver/x86/nvmm_x86_svm.c +++ b/src/driver/x86/nvmm_x86_svm.c @@ -253,7 +253,7 @@ struct vmcb_ctrl { #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) -#define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) +#define VMCB_CTRL_INTERCEPT_CR0_SEL __BIT(5) #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) @@ -503,6 +503,7 @@ static os_mtx_t svm_asidlock __cacheline_aligned; static bool svm_decode_assist __read_mostly; static uint32_t svm_ctrl_tlb_flush __read_mostly; +static bool svm_cr0quirk_supported __read_mostly; #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) static uint64_t svm_xcr0_mask __read_mostly; @@ -574,6 +575,11 @@ struct svm_cpudata { uint64_t gtsc; struct nvmm_x86_xsave gxsave __aligned(64); + /* CR0 quirk support. */ + bool cr0quirk_is_active; + bool cr0quirk_want_reinstate; + uint64_t gcr0_cd_nw; + /* Limits. */ uint64_t cr4_valid; uint64_t efer_valid; @@ -1047,9 +1053,176 @@ svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason) exit->reason = reason; } +static inline uint64_t +svm_read_gcr0(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + uint64_t fakecr0; + + fakecr0 = vmcb->state.cr0 | cpudata->gcr0_cd_nw; + + return fakecr0; +} + +static void +svm_write_gcr0(struct nvmm_cpu *vcpu, uint64_t fakecr0) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + uint64_t realcr0; + + realcr0 = fakecr0; + + if (__predict_true(svm_cr0quirk_supported)) { + /* + * fakecr0 is the value the guest believes is in %cr0. realcr0 + * is the actual value in %cr0. + * + * In realcr0 we force CR0_NW and CR0_CD to 0. + * + * If fakecr0 doesn't match realcr0, then enable the read-CR0 + * intercept so that we return fakecr0 if the guest reads CR0. + * If fakecr0 matches realcr0 and the read-CR0 intercept is + * enabled, then disable it and have the hardware return the + * CR0 value natively. + */ + cpudata->gcr0_cd_nw = fakecr0 & (CR0_CD | CR0_NW); + realcr0 = fakecr0 & ~(CR0_CD | CR0_NW); + } + + if (realcr0 != fakecr0) { + if (!cpudata->cr0quirk_is_active) { + vmcb->ctrl.intercept_cr |= VMCB_CTRL_INTERCEPT_RCR(0); + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); + cpudata->cr0quirk_is_active = true; + } + } else if (__predict_false(cpudata->cr0quirk_is_active)) { + vmcb->ctrl.intercept_cr &= ~VMCB_CTRL_INTERCEPT_RCR(0); + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); + cpudata->cr0quirk_is_active = false; + } + + vmcb->state.cr0 = realcr0; +} + +static void +svm_abandon_cr0_quirk(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + + vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_CR0_SEL; + vmcb->ctrl.intercept_cr &= ~VMCB_CTRL_INTERCEPT_RCR(0); + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); + + vmcb->state.cr0 |= cpudata->gcr0_cd_nw; + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); + + cpudata->cr0quirk_is_active = false; + cpudata->cr0quirk_want_reinstate = true; + cpudata->gcr0_cd_nw = 0; +} + +static void +svm_reinstate_cr0_quirk(struct nvmm_cpu *vcpu) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + + cpudata->cr0quirk_want_reinstate = false; + + vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_CR0_SEL; + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); + + svm_write_gcr0(vcpu, svm_read_gcr0(vcpu)); + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); +} + #define SVM_EXIT_CRDR_GPR __BITS(3,0) #define SVM_EXIT_CRDR_CR __BIT(63) +static void +svm_exit_rcr0(struct nvmm_cpu *vcpu, struct nvmm_vcpu_exit *exit) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + uint64_t info, gpr, fakecr0; + bool is_mov; + + OS_ASSERT(cpudata->cr0quirk_is_active); + + info = vmcb->ctrl.exitinfo1; + gpr = __SHIFTOUT(info, SVM_EXIT_CRDR_GPR); + is_mov = __SHIFTOUT(info, SVM_EXIT_CRDR_CR); + + if (!is_mov) { + /* + * If this is an SMSW instruction, abandon the CR0 quirk, and + * return to the guest without advancing RIP. The quirk will + * be re-instated on the next VCPU iteration. + */ + svm_abandon_cr0_quirk(vcpu); + exit->reason = NVMM_VCPU_EXIT_NONE; + return; + } + + fakecr0 = svm_read_gcr0(vcpu); + if (gpr == NVMM_X64_GPR_RAX) { + vmcb->state.rax = fakecr0; + } else if (gpr == NVMM_X64_GPR_RSP) { + vmcb->state.rsp = fakecr0; + } else { + cpudata->gprs[gpr] = fakecr0; + } + + svm_inkernel_advance(vmcb); + exit->reason = NVMM_VCPU_EXIT_NONE; +} + +static void +svm_exit_wcr0_sel(struct nvmm_cpu *vcpu, struct nvmm_vcpu_exit *exit) +{ + struct svm_cpudata *cpudata = vcpu->cpudata; + struct vmcb *vmcb = cpudata->vmcb; + uint64_t info, gpr, fakecr0; + bool is_mov; + + info = vmcb->ctrl.exitinfo1; + gpr = __SHIFTOUT(info, SVM_EXIT_CRDR_GPR); + is_mov = __SHIFTOUT(info, SVM_EXIT_CRDR_CR); + + if (!is_mov) { + /* + * If this is an LMSW instruction, abandon the CR0 quirk, and + * return to the guest without advancing RIP. The quirk will + * be re-instated on the next VCPU iteration. + */ + svm_abandon_cr0_quirk(vcpu); + exit->reason = NVMM_VCPU_EXIT_NONE; + return; + } + + if (gpr == NVMM_X64_GPR_RAX) { + fakecr0 = vmcb->state.rax; + } else if (gpr == NVMM_X64_GPR_RSP) { + fakecr0 = vmcb->state.rsp; + } else { + fakecr0 = cpudata->gprs[gpr]; + } + + if ((svm_read_gcr0(vcpu) ^ fakecr0) & CR0_TLB_FLUSH) { + cpudata->gtlb_want_flush = true; + } + + svm_write_gcr0(vcpu, fakecr0); + + svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); + + svm_inkernel_advance(vmcb); + exit->reason = NVMM_VCPU_EXIT_NONE; +} + static void svm_exit_wcr4(struct nvmm_cpu *vcpu, struct nvmm_vcpu_exit *exit) { @@ -1606,6 +1779,10 @@ svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, svm_vcpu_guest_dbregs_enter(vcpu); svm_vcpu_guest_misc_enter(vcpu); + if (__predict_false(cpudata->cr0quirk_want_reinstate)) { + svm_reinstate_cr0_quirk(vcpu); + } + while (1) { if (cpudata->gtlb_want_flush) { vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; @@ -1636,6 +1813,12 @@ svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, svm_exit_evt(cpudata, vmcb); switch (vmcb->ctrl.exitcode) { + case VMCB_EXITCODE_CR0_READ: + svm_exit_rcr0(vcpu, exit); + break; + case VMCB_EXITCODE_CR0_SEL_WRITE: + svm_exit_wcr0_sel(vcpu, exit); + break; case VMCB_EXITCODE_CR4_WRITE: svm_exit_wcr4(vcpu, exit); break; @@ -1891,7 +2074,7 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu) } if (flags & NVMM_X64_STATE_CRS) { - vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; + svm_write_gcr0(vcpu, state->crs[NVMM_X64_CR_CR0]); vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4] & @@ -2028,7 +2211,7 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu) } if (flags & NVMM_X64_STATE_CRS) { - state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; + state->crs[NVMM_X64_CR_CR0] = svm_read_gcr0(vcpu); state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; @@ -2227,7 +2410,7 @@ svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) * Allow: * - SMI [smm interrupts] * - VINTR [virtual interrupts] - * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] + * - CR0_SEL [selective CR0 writes, see note below] * - RIDTR [reads of IDTR] * - RGDTR [reads of GDTR] * - RLDTR [reads of LDTR] @@ -2245,7 +2428,8 @@ svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) * - INVLPG [invplg instruction] * - TASKSW [task switches] * - * Intercept the rest below. + * Intercept the rest below. Conditionally intercept CR0_SEL if the + * CR0 quirk is supported. */ vmcb->ctrl.intercept_misc1 = VMCB_CTRL_INTERCEPT_INTR | @@ -2261,6 +2445,9 @@ svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) VMCB_CTRL_INTERCEPT_MSR_PROT | VMCB_CTRL_INTERCEPT_FERR_FREEZE | VMCB_CTRL_INTERCEPT_SHUTDOWN; + if (svm_cr0quirk_supported) { + vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_CR0_SEL; + } /* * Allow: @@ -2586,6 +2773,14 @@ svm_ident(void) svm_decode_assist = (descs.edx & CPUID_8_0A_EDX_DecodeAssists) != 0; + /* + * If DecodeAssist is supported, we implement a CR0 quirk to force + * CR0.CD=0 and CR0.NW=0 in the guest, for symmetry with VMX, and in + * order to increase performance in the pathological cases where the + * guest's UEFI bootloader rides with the cache disabled. + */ + svm_cr0quirk_supported = svm_decode_assist; + msr = rdmsr(MSR_VM_CR); if ((msr & VM_CR_SVMED) && (msr & VM_CR_LOCK)) { os_printf("nvmm: SVM disabled in BIOS\n");