Index: amd64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/cpu.c,v diff -u -p -r1.196 cpu.c --- amd64/cpu.c 5 Jun 2025 09:29:54 -0000 1.196 +++ amd64/cpu.c 5 Jul 2025 10:53:24 -0000 @@ -637,6 +637,8 @@ cpu_attach(struct device *parent, struct #endif #if defined(MULTIPROCESSOR) + cpu_xcall_establish(ci); + /* * Allocate UPAGES contiguous pages for the idle PCB and stack. */ Index: amd64/ipifuncs.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/ipifuncs.c,v diff -u -p -r1.39 ipifuncs.c --- amd64/ipifuncs.c 7 Jun 2024 16:53:35 -0000 1.39 +++ amd64/ipifuncs.c 5 Jul 2025 10:53:24 -0000 @@ -108,6 +108,7 @@ void (*ipifunc[X86_NIPI])(struct cpu_inf NULL, #endif x86_64_ipi_wbinvd, + cpu_xcall_dispatch, }; void Index: amd64/vmm_machdep.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/vmm_machdep.c,v diff -u -p -r1.61 vmm_machdep.c --- amd64/vmm_machdep.c 4 Jul 2025 10:11:28 -0000 1.61 +++ amd64/vmm_machdep.c 5 Jul 2025 10:53:24 -0000 @@ -156,9 +156,7 @@ int vmm_update_pvclock(struct vcpu *); void vmm_pv_wall_clock(struct vcpu *, paddr_t); int vmm_pat_is_valid(uint64_t); -#ifdef MULTIPROCESSOR static int vmx_remote_vmclear(struct cpu_info*, struct vcpu *); -#endif #ifdef VMM_DEBUG void vmx_vcpu_dump_regs(struct vcpu *); @@ -367,25 +365,11 @@ vmm_quiesce_vmx(void) continue; } -#ifdef MULTIPROCESSOR - if (vcpu->vc_last_pcpu != curcpu()) { - /* Remote cpu vmclear via ipi. */ - err = vmx_remote_vmclear(vcpu->vc_last_pcpu, - vcpu); - if (err) - printf("%s: failed to remote vmclear " - "vcpu %d of vm %d\n", __func__, - vcpu->vc_id, vm->vm_id); - } else -#endif - { - /* Local cpu vmclear instruction. */ - if ((err = vmclear(&vcpu->vc_control_pa))) - printf("%s: failed to locally vmclear " - "vcpu %d of vm %d\n", __func__, - vcpu->vc_id, vm->vm_id); - atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, - VMCS_CLEARED); + err = vmx_remote_vmclear(vcpu->vc_last_pcpu, vcpu); + if (err) { + printf("%s: failed to vmclear " + "vcpu %d of vm %d\n", __func__, + vcpu->vc_id, vm->vm_id); } rw_exit_write(&vcpu->vc_lock); @@ -626,18 +610,24 @@ out: * * Starts VMM mode on the system */ + +static void +vmm_start_xcall(void *arg) +{ + struct refcnt *r = arg; + start_vmm_on_cpu(curcpu()); + refcnt_rele_wake(r); +} + int vmm_start(void) { int rv = 0; struct cpu_info *self = curcpu(); -#ifdef MULTIPROCESSOR struct cpu_info *ci; CPU_INFO_ITERATOR cii; -#ifdef MP_LOCKDEBUG - long nticks; -#endif /* MP_LOCKDEBUG */ -#endif /* MULTIPROCESSOR */ + struct refcnt r = REFCNT_INITIALIZER(); + struct xcall xc = XCALL_INITIALIZER(vmm_start_xcall, &r); rw_enter_write(&vmm_softc->sc_slock); @@ -645,8 +635,12 @@ vmm_start(void) if (self->ci_flags & CPUF_VMM) goto unlock; - /* Start VMM on this CPU */ - start_vmm_on_cpu(self); + CPU_INFO_FOREACH(cii, ci) { + refcnt_take(&r); + cpu_xcall(ci, &xc); + } + refcnt_finalize(&r, "vmmstart"); + if (!(self->ci_flags & CPUF_VMM)) { printf("%s: failed to enter VMM mode\n", self->ci_dev->dv_xname); @@ -654,28 +648,6 @@ vmm_start(void) goto unlock; } -#ifdef MULTIPROCESSOR - /* Broadcast start VMM IPI */ - x86_broadcast_ipi(X86_IPI_START_VMM); - - CPU_INFO_FOREACH(cii, ci) { - if (ci == self) - continue; -#ifdef MP_LOCKDEBUG - nticks = __mp_lock_spinout; -#endif /* MP_LOCKDEBUG */ - while (!(ci->ci_flags & CPUF_VMM)) { - CPU_BUSY_CYCLE(); -#ifdef MP_LOCKDEBUG - if (--nticks <= 0) { - db_printf("%s: spun out", __func__); - db_enter(); - nticks = __mp_lock_spinout; - } -#endif /* MP_LOCKDEBUG */ - } - } -#endif /* MULTIPROCESSOR */ unlock: rw_exit_write(&vmm_softc->sc_slock); return (rv); @@ -686,18 +658,24 @@ unlock: * * Stops VMM mode on the system */ + +static void +vmm_stop_xcall(void *arg) +{ + struct refcnt *r = arg; + stop_vmm_on_cpu(curcpu()); + refcnt_rele_wake(r); +} + int vmm_stop(void) { int rv = 0; struct cpu_info *self = curcpu(); -#ifdef MULTIPROCESSOR struct cpu_info *ci; CPU_INFO_ITERATOR cii; -#ifdef MP_LOCKDEBUG - long nticks; -#endif /* MP_LOCKDEBUG */ -#endif /* MULTIPROCESSOR */ + struct refcnt r = REFCNT_INITIALIZER(); + struct xcall xc = XCALL_INITIALIZER(vmm_stop_xcall, &r); rw_enter_write(&vmm_softc->sc_slock); @@ -705,40 +683,21 @@ vmm_stop(void) if (!(self->ci_flags & CPUF_VMM)) goto unlock; - /* Stop VMM on this CPU */ - stop_vmm_on_cpu(self); + CPU_INFO_FOREACH(cii, ci) { + refcnt_take(&r); + cpu_xcall(ci, &xc); + } + refcnt_finalize(&r, "vmmstop"); + if (self->ci_flags & CPUF_VMM) { printf("%s: failed to exit VMM mode\n", self->ci_dev->dv_xname); rv = EIO; goto unlock; } - -#ifdef MULTIPROCESSOR - /* Stop VMM on other CPUs */ - x86_broadcast_ipi(X86_IPI_STOP_VMM); - - CPU_INFO_FOREACH(cii, ci) { - if (ci == self) - continue; -#ifdef MP_LOCKDEBUG - nticks = __mp_lock_spinout; -#endif /* MP_LOCKDEBUG */ - while ((ci->ci_flags & CPUF_VMM)) { - CPU_BUSY_CYCLE(); -#ifdef MP_LOCKDEBUG - if (--nticks <= 0) { - db_printf("%s: spunout", __func__); - db_enter(); - nticks = __mp_lock_spinout; - } -#endif /* MP_LOCKDEBUG */ - } - } -#endif /* MULTIPROCESSOR */ unlock: rw_exit_write(&vmm_softc->sc_slock); - return (0); + return (rv); } /* @@ -863,34 +822,26 @@ vmclear_on_cpu(struct cpu_info *ci) } } -#ifdef MULTIPROCESSOR +void +vmx_remote_vmclear_xcall(void *arg) +{ + struct cpu_info *ci = curcpu(); + + if ((ci->ci_flags & CPUF_VMM) && (ci->ci_vmm_flags & CI_VMM_VMX)) { + if (vmclear(arg)) + panic("VMCLEAR xcall failed"); + } +} + static int vmx_remote_vmclear(struct cpu_info *ci, struct vcpu *vcpu) { -#ifdef MP_LOCKDEBUG - long nticks = __mp_lock_spinout; -#endif /* MP_LOCKDEBUG */ - - rw_enter_write(&ci->ci_vmcs_lock); - atomic_swap_ulong(&ci->ci_vmcs_pa, vcpu->vc_control_pa); - x86_send_ipi(ci, X86_IPI_VMCLEAR_VMM); - - while (ci->ci_vmcs_pa != VMX_VMCS_PA_CLEAR) { - CPU_BUSY_CYCLE(); -#ifdef MP_LOCKDEBUG - if (--nticks <= 0) { - db_printf("%s: spun out\n", __func__); - db_enter(); - nticks = __mp_lock_spinout; - } -#endif /* MP_LOCKDEBUG */ - } + cpu_xcall_sync(ci, vmx_remote_vmclear_xcall, &vcpu->vc_control_pa, + "vmclear"); atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_CLEARED); - rw_exit_write(&ci->ci_vmcs_lock); return (0); } -#endif /* MULTIPROCESSOR */ /* * vm_impl_init @@ -948,26 +899,19 @@ vm_impl_deinit(struct vm *vm) int vcpu_reload_vmcs_vmx(struct vcpu *vcpu) { - struct cpu_info *ci, *last_ci; + struct cpu_info *ci; rw_assert_wrlock(&vcpu->vc_lock); - ci = curcpu(); - last_ci = vcpu->vc_last_pcpu; - - if (last_ci == NULL) { + ci = vcpu->vc_last_pcpu; + if (ci == NULL) { /* First launch */ - if (vmclear(&vcpu->vc_control_pa)) - return (EINVAL); - atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_CLEARED); -#ifdef MULTIPROCESSOR - } else if (last_ci != ci) { - /* We've moved CPUs at some point, so remote VMCLEAR */ - if (vmx_remote_vmclear(last_ci, vcpu)) - return (EINVAL); - KASSERT(vcpu->vc_vmx_vmcs_state == VMCS_CLEARED); -#endif /* MULTIPROCESSOR */ + ci = curcpu(); } + + if (vmx_remote_vmclear(ci, vcpu)) + return (EINVAL); + KASSERT(vcpu->vc_vmx_vmcs_state == VMCS_CLEARED); if (vmptrld(&vcpu->vc_control_pa)) { printf("%s: vmptrld\n", __func__); Index: conf/files.amd64 =================================================================== RCS file: /cvs/src/sys/arch/amd64/conf/files.amd64,v diff -u -p -r1.115 files.amd64 --- conf/files.amd64 7 Jun 2025 15:11:12 -0000 1.115 +++ conf/files.amd64 5 Jul 2025 10:53:24 -0000 @@ -94,7 +94,7 @@ file arch/amd64/amd64/mpbios.c mpbios n file arch/amd64/amd64/mpbios_intr_fixup.c mpbios & pci define cpu {[apid = -1]} -device cpu +device cpu: xcall attach cpu at mainbus file arch/amd64/amd64/cpu.c cpu Index: include/cpu.h =================================================================== RCS file: /cvs/src/sys/arch/amd64/include/cpu.h,v diff -u -p -r1.180 cpu.h --- include/cpu.h 28 Apr 2025 16:18:25 -0000 1.180 +++ include/cpu.h 5 Jul 2025 10:53:24 -0000 @@ -53,6 +53,7 @@ #include #include #include +#include #include #ifdef _KERNEL @@ -215,6 +216,7 @@ struct cpu_info { #ifdef MULTIPROCESSOR struct srp_hazard ci_srp_hazards[SRP_HAZARD_NUM]; + struct xcall_cpu ci_xcall; #define __HAVE_UVM_PERCPU struct uvm_pmr_cache ci_uvm; /* [o] page cache */ #endif Index: include/intr.h =================================================================== RCS file: /cvs/src/sys/arch/amd64/include/intr.h,v diff -u -p -r1.36 intr.h --- include/intr.h 11 Jun 2025 09:57:01 -0000 1.36 +++ include/intr.h 5 Jul 2025 10:53:24 -0000 @@ -222,7 +222,9 @@ void x86_ipi_handler(void); void x86_setperf_ipi(struct cpu_info *); extern void (*ipifunc[X86_NIPI])(struct cpu_info *); -#endif + +#define cpu_xcall_ipi(_ci) x86_send_ipi((_ci), X86_IPI_XCALL) +#endif /* MULTIPROCESSOR */ #endif /* !_LOCORE */ Index: include/intrdefs.h =================================================================== RCS file: /cvs/src/sys/arch/amd64/include/intrdefs.h,v diff -u -p -r1.24 intrdefs.h --- include/intrdefs.h 26 May 2024 13:37:31 -0000 1.24 +++ include/intrdefs.h 5 Jul 2025 10:53:24 -0000 @@ -85,8 +85,9 @@ #define X86_IPI_START_VMM 0x00000100 #define X86_IPI_STOP_VMM 0x00000200 #define X86_IPI_WBINVD 0x00000400 +#define X86_IPI_XCALL 0x00000800 -#define X86_NIPI 12 +#define X86_NIPI 13 #define IREENT_MAGIC 0x18041969