commit 758d9fd121c59729955b3bf5c06f953e54ae67d9
Author: m00nbsd <42475391+m00nbsd@users.noreply.github.com>
Date:   Mon Jul 27 20:53:10 2026 -0400

    nvmm_x86_svm: implement a CR0 quirk
    
    Some guests have issues where their UEFI bootloader rides with CR0.CD=1
    which is extremely slow under SVM. Implement a quirk to force CR0.CD=0
    in the guest, akin to KVM's KVM_X86_QUIRK_CD_NW_CLEARED.
    
    If the guest attempts to execute an SMSW, or an LMSW when the quirk is
    active, we abandon the quirk, let the guest execute SMSW/LMSW natively,
    and we passively reinstate the quirk in the next VCPU iteration. This
    is to avoid doing heavy decoding and emulation of SMSW/LMSW.

diff --git a/src/driver/x86/nvmm_x86_svm.c b/src/driver/x86/nvmm_x86_svm.c
index edc97df..bbb2485 100644
--- a/src/driver/x86/nvmm_x86_svm.c
+++ b/src/driver/x86/nvmm_x86_svm.c
@@ -253,7 +253,7 @@ struct vmcb_ctrl {
 #define VMCB_CTRL_INTERCEPT_SMI		__BIT(2)
 #define VMCB_CTRL_INTERCEPT_INIT	__BIT(3)
 #define VMCB_CTRL_INTERCEPT_VINTR	__BIT(4)
-#define VMCB_CTRL_INTERCEPT_CR0_SPEC	__BIT(5)
+#define VMCB_CTRL_INTERCEPT_CR0_SEL	__BIT(5)
 #define VMCB_CTRL_INTERCEPT_RIDTR	__BIT(6)
 #define VMCB_CTRL_INTERCEPT_RGDTR	__BIT(7)
 #define VMCB_CTRL_INTERCEPT_RLDTR	__BIT(8)
@@ -503,6 +503,7 @@ static os_mtx_t svm_asidlock __cacheline_aligned;
 
 static bool svm_decode_assist __read_mostly;
 static uint32_t svm_ctrl_tlb_flush __read_mostly;
+static bool svm_cr0quirk_supported __read_mostly;
 
 #define SVM_XCR0_MASK_DEFAULT	(XCR0_X87|XCR0_SSE)
 static uint64_t svm_xcr0_mask __read_mostly;
@@ -574,6 +575,11 @@ struct svm_cpudata {
 	uint64_t gtsc;
 	struct nvmm_x86_xsave gxsave __aligned(64);
 
+	/* CR0 quirk support. */
+	bool cr0quirk_is_active;
+	bool cr0quirk_want_reinstate;
+	uint64_t gcr0_cd_nw;
+
 	/* Limits. */
 	uint64_t cr4_valid;
 	uint64_t efer_valid;
@@ -1047,9 +1053,176 @@ svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason)
 	exit->reason = reason;
 }
 
+static inline uint64_t
+svm_read_gcr0(struct nvmm_cpu *vcpu)
+{
+	struct svm_cpudata *cpudata = vcpu->cpudata;
+	struct vmcb *vmcb = cpudata->vmcb;
+	uint64_t fakecr0;
+
+	fakecr0 = vmcb->state.cr0 | cpudata->gcr0_cd_nw;
+
+	return fakecr0;
+}
+
+static void
+svm_write_gcr0(struct nvmm_cpu *vcpu, uint64_t fakecr0)
+{
+	struct svm_cpudata *cpudata = vcpu->cpudata;
+	struct vmcb *vmcb = cpudata->vmcb;
+	uint64_t realcr0;
+
+	realcr0 = fakecr0;
+
+	if (__predict_true(svm_cr0quirk_supported)) {
+		/*
+		 * fakecr0 is the value the guest believes is in %cr0. realcr0
+		 * is the actual value in %cr0.
+		 *
+		 * In realcr0 we force CR0_NW and CR0_CD to 0.
+		 *
+		 * If fakecr0 doesn't match realcr0, then enable the read-CR0
+		 * intercept so that we return fakecr0 if the guest reads CR0.
+		 * If fakecr0 matches realcr0 and the read-CR0 intercept is
+		 * enabled, then disable it and have the hardware return the
+		 * CR0 value natively.
+		 */
+		cpudata->gcr0_cd_nw = fakecr0 & (CR0_CD | CR0_NW);
+		realcr0 = fakecr0 & ~(CR0_CD | CR0_NW);
+	}
+
+	if (realcr0 != fakecr0) {
+		if (!cpudata->cr0quirk_is_active) {
+			vmcb->ctrl.intercept_cr |= VMCB_CTRL_INTERCEPT_RCR(0);
+			svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I);
+			cpudata->cr0quirk_is_active = true;
+		}
+	} else if (__predict_false(cpudata->cr0quirk_is_active)) {
+		vmcb->ctrl.intercept_cr &= ~VMCB_CTRL_INTERCEPT_RCR(0);
+		svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I);
+		cpudata->cr0quirk_is_active = false;
+	}
+
+	vmcb->state.cr0 = realcr0;
+}
+
+static void
+svm_abandon_cr0_quirk(struct nvmm_cpu *vcpu)
+{
+	struct svm_cpudata *cpudata = vcpu->cpudata;
+	struct vmcb *vmcb = cpudata->vmcb;
+
+	vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_CR0_SEL;
+	vmcb->ctrl.intercept_cr &= ~VMCB_CTRL_INTERCEPT_RCR(0);
+	svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I);
+
+	vmcb->state.cr0 |= cpudata->gcr0_cd_nw;
+	svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR);
+
+	cpudata->cr0quirk_is_active = false;
+	cpudata->cr0quirk_want_reinstate = true;
+	cpudata->gcr0_cd_nw = 0;
+}
+
+static void
+svm_reinstate_cr0_quirk(struct nvmm_cpu *vcpu)
+{
+	struct svm_cpudata *cpudata = vcpu->cpudata;
+	struct vmcb *vmcb = cpudata->vmcb;
+
+	cpudata->cr0quirk_want_reinstate = false;
+
+	vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_CR0_SEL;
+	svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I);
+
+	svm_write_gcr0(vcpu, svm_read_gcr0(vcpu));
+	svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR);
+}
+
 #define SVM_EXIT_CRDR_GPR	__BITS(3,0)
 #define SVM_EXIT_CRDR_CR	__BIT(63)
 
+static void
+svm_exit_rcr0(struct nvmm_cpu *vcpu, struct nvmm_vcpu_exit *exit)
+{
+	struct svm_cpudata *cpudata = vcpu->cpudata;
+	struct vmcb *vmcb = cpudata->vmcb;
+	uint64_t info, gpr, fakecr0;
+	bool is_mov;
+
+	OS_ASSERT(cpudata->cr0quirk_is_active);
+
+	info = vmcb->ctrl.exitinfo1;
+	gpr = __SHIFTOUT(info, SVM_EXIT_CRDR_GPR);
+	is_mov = __SHIFTOUT(info, SVM_EXIT_CRDR_CR);
+
+	if (!is_mov) {
+		/*
+		 * If this is an SMSW instruction, abandon the CR0 quirk, and
+		 * return to the guest without advancing RIP. The quirk will
+		 * be re-instated on the next VCPU iteration.
+		 */
+		svm_abandon_cr0_quirk(vcpu);
+		exit->reason = NVMM_VCPU_EXIT_NONE;
+		return;
+	}
+
+	fakecr0 = svm_read_gcr0(vcpu);
+	if (gpr == NVMM_X64_GPR_RAX) {
+		vmcb->state.rax = fakecr0;
+	} else if (gpr == NVMM_X64_GPR_RSP) {
+		vmcb->state.rsp = fakecr0;
+	} else {
+		cpudata->gprs[gpr] = fakecr0;
+	}
+
+	svm_inkernel_advance(vmcb);
+	exit->reason = NVMM_VCPU_EXIT_NONE;
+}
+
+static void
+svm_exit_wcr0_sel(struct nvmm_cpu *vcpu, struct nvmm_vcpu_exit *exit)
+{
+	struct svm_cpudata *cpudata = vcpu->cpudata;
+	struct vmcb *vmcb = cpudata->vmcb;
+	uint64_t info, gpr, fakecr0;
+	bool is_mov;
+
+	info = vmcb->ctrl.exitinfo1;
+	gpr = __SHIFTOUT(info, SVM_EXIT_CRDR_GPR);
+	is_mov = __SHIFTOUT(info, SVM_EXIT_CRDR_CR);
+
+	if (!is_mov) {
+		/*
+		 * If this is an LMSW instruction, abandon the CR0 quirk, and
+		 * return to the guest without advancing RIP. The quirk will
+		 * be re-instated on the next VCPU iteration.
+		 */
+		svm_abandon_cr0_quirk(vcpu);
+		exit->reason = NVMM_VCPU_EXIT_NONE;
+		return;
+	}
+
+	if (gpr == NVMM_X64_GPR_RAX) {
+		fakecr0 = vmcb->state.rax;
+	} else if (gpr == NVMM_X64_GPR_RSP) {
+		fakecr0 = vmcb->state.rsp;
+	} else {
+		fakecr0 = cpudata->gprs[gpr];
+	}
+
+	if ((svm_read_gcr0(vcpu) ^ fakecr0) & CR0_TLB_FLUSH) {
+		cpudata->gtlb_want_flush = true;
+	}
+
+	svm_write_gcr0(vcpu, fakecr0);
+
+	svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR);
+
+	svm_inkernel_advance(vmcb);
+	exit->reason = NVMM_VCPU_EXIT_NONE;
+}
+
 static void
 svm_exit_wcr4(struct nvmm_cpu *vcpu, struct nvmm_vcpu_exit *exit)
 {
@@ -1606,6 +1779,10 @@ svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
 	svm_vcpu_guest_dbregs_enter(vcpu);
 	svm_vcpu_guest_misc_enter(vcpu);
 
+	if (__predict_false(cpudata->cr0quirk_want_reinstate)) {
+		svm_reinstate_cr0_quirk(vcpu);
+	}
+
 	while (1) {
 		if (cpudata->gtlb_want_flush) {
 			vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush;
@@ -1636,6 +1813,12 @@ svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
 		svm_exit_evt(cpudata, vmcb);
 
 		switch (vmcb->ctrl.exitcode) {
+		case VMCB_EXITCODE_CR0_READ:
+			svm_exit_rcr0(vcpu, exit);
+			break;
+		case VMCB_EXITCODE_CR0_SEL_WRITE:
+			svm_exit_wcr0_sel(vcpu, exit);
+			break;
 		case VMCB_EXITCODE_CR4_WRITE:
 			svm_exit_wcr4(vcpu, exit);
 			break;
@@ -1891,7 +2074,7 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu)
 	}
 
 	if (flags & NVMM_X64_STATE_CRS) {
-		vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0];
+		svm_write_gcr0(vcpu, state->crs[NVMM_X64_CR_CR0]);
 		vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2];
 		vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3];
 		vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4] &
@@ -2028,7 +2211,7 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu)
 	}
 
 	if (flags & NVMM_X64_STATE_CRS) {
-		state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0;
+		state->crs[NVMM_X64_CR_CR0] = svm_read_gcr0(vcpu);
 		state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2;
 		state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3;
 		state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4;
@@ -2227,7 +2410,7 @@ svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
 	 * Allow:
 	 *  - SMI [smm interrupts]
 	 *  - VINTR [virtual interrupts]
-	 *  - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP]
+	 *  - CR0_SEL [selective CR0 writes, see note below]
 	 *  - RIDTR [reads of IDTR]
 	 *  - RGDTR [reads of GDTR]
 	 *  - RLDTR [reads of LDTR]
@@ -2245,7 +2428,8 @@ svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
 	 *  - INVLPG [invplg instruction]
 	 *  - TASKSW [task switches]
 	 *
-	 * Intercept the rest below.
+	 * Intercept the rest below. Conditionally intercept CR0_SEL if the
+	 * CR0 quirk is supported.
 	 */
 	vmcb->ctrl.intercept_misc1 =
 	    VMCB_CTRL_INTERCEPT_INTR |
@@ -2261,6 +2445,9 @@ svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
 	    VMCB_CTRL_INTERCEPT_MSR_PROT |
 	    VMCB_CTRL_INTERCEPT_FERR_FREEZE |
 	    VMCB_CTRL_INTERCEPT_SHUTDOWN;
+	if (svm_cr0quirk_supported) {
+		vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_CR0_SEL;
+	}
 
 	/*
 	 * Allow:
@@ -2586,6 +2773,14 @@ svm_ident(void)
 
 	svm_decode_assist = (descs.edx & CPUID_8_0A_EDX_DecodeAssists) != 0;
 
+	/*
+	 * If DecodeAssist is supported, we implement a CR0 quirk to force
+	 * CR0.CD=0 and CR0.NW=0 in the guest, for symmetry with VMX, and in
+	 * order to increase performance in the pathological cases where the
+	 * guest's UEFI bootloader rides with the cache disabled.
+	 */
+	svm_cr0quirk_supported = svm_decode_assist;
+
 	msr = rdmsr(MSR_VM_CR);
 	if ((msr & VM_CR_SVMED) && (msr & VM_CR_LOCK)) {
 		os_printf("nvmm: SVM disabled in BIOS\n");
