Index: arch/amd64/amd64/cpu.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/cpu.c,v
retrieving revision 1.102
diff -u -p -r1.102 cpu.c
--- arch/amd64/amd64/cpu.c	28 Jul 2016 21:57:57 -0000	1.102
+++ arch/amd64/amd64/cpu.c	19 Jan 2017 05:32:46 -0000
@@ -462,6 +462,9 @@ cpu_attach(struct device *parent, struct
 	cpu_vm_init(ci);
 
 #if defined(MULTIPROCESSOR)
+	mtx_init(&ci->ci_xcall_mtx, IPL_HIGH);
+	TAILQ_INIT(&ci->ci_xcall_list);
+
 	if (mp_verbose) {
 		printf("%s: kstack at 0x%lx for %d bytes\n",
 		    sc->sc_dev.dv_xname, kstack, USPACE);
Index: arch/amd64/amd64/intr.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/intr.c,v
retrieving revision 1.48
diff -u -p -r1.48 intr.c
--- arch/amd64/amd64/intr.c	22 Jun 2016 01:12:38 -0000	1.48
+++ arch/amd64/amd64/intr.c	19 Jan 2017 05:32:46 -0000
@@ -550,7 +550,10 @@ struct intrhand fake_softclock_intrhand;
 struct intrhand fake_softnet_intrhand;
 struct intrhand fake_softtty_intrhand;
 struct intrhand fake_timer_intrhand;
+#ifdef MULTIPROCESSOR
 struct intrhand fake_ipi_intrhand;
+struct intrhand fake_xcall_intrhand;
+#endif
 #if NXEN > 0
 struct intrhand fake_xen_intrhand;
 #endif
@@ -621,6 +624,15 @@ cpu_intr_init(struct cpu_info *ci)
 	isp->is_handlers = &fake_ipi_intrhand;
 	isp->is_pic = &local_pic;
 	ci->ci_isources[LIR_IPI] = isp;
+	isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
+	if (isp == NULL)
+		panic("can't allocate fixed interrupt source");
+	isp->is_recurse = Xxcallintr;
+	isp->is_resume = Xxcallintr;
+	fake_xcall_intrhand.ih_level = IPL_SOFTCLOCK;
+	isp->is_handlers = &fake_xcall_intrhand;
+	isp->is_pic = &local_pic;
+	ci->ci_isources[SIR_XCALL] = isp;
 #endif
 #if NXEN > 0
 	isp = malloc(sizeof (struct intrsource), M_DEVBUF, M_NOWAIT|M_ZERO);
@@ -741,6 +753,5 @@ softintr(int sir)
 {
 	struct cpu_info *ci = curcpu();
 
-	__asm volatile("lock; orq %1, %0" :
-	    "=m"(ci->ci_ipending) : "ir" (1UL << sir));
+	x86_atomic_setbits_u64(&ci->ci_ipending, 1UL << sir);
 }
Index: arch/amd64/amd64/ipifuncs.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/ipifuncs.c,v
retrieving revision 1.28
diff -u -p -r1.28 ipifuncs.c
--- arch/amd64/amd64/ipifuncs.c	23 Nov 2015 22:57:12 -0000	1.28
+++ arch/amd64/amd64/ipifuncs.c	19 Jan 2017 05:32:46 -0000
@@ -65,6 +65,8 @@ void x86_64_ipi_halt(struct cpu_info *);
 void x86_64_ipi_synch_fpu(struct cpu_info *);
 void x86_64_ipi_flush_fpu(struct cpu_info *);
 
+void x86_64_ipi_xcall(struct cpu_info *);
+
 #if NVMM > 0
 void x86_64_ipi_start_vmm(struct cpu_info *);
 void x86_64_ipi_stop_vmm(struct cpu_info *);
@@ -102,6 +104,7 @@ void (*ipifunc[X86_NIPI])(struct cpu_inf
 	NULL,
 	NULL,
 #endif
+	x86_64_ipi_xcall,
 };
 
 void
@@ -163,3 +166,13 @@ x86_64_ipi_stop_vmm(struct cpu_info *ci)
 	stop_vmm_on_cpu(ci);
 }
 #endif /* NVMM > 0 */
+
+void
+x86_64_ipi_xcall(struct cpu_info *ci)
+{
+	/*
+	 * this is an inlining of softintr() because we already have
+	 * curcpu() and the SIR_XCALL bit to set.
+	 */
+	x86_atomic_setbits_u64(&ci->ci_ipending, 1UL << SIR_XCALL);
+};
Index: arch/amd64/amd64/softintr.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/softintr.c,v
retrieving revision 1.9
diff -u -p -r1.9 softintr.c
--- arch/amd64/amd64/softintr.c	28 Aug 2015 00:03:53 -0000	1.9
+++ arch/amd64/amd64/softintr.c	19 Jan 2017 05:32:46 -0000
@@ -168,3 +168,43 @@ softintr_disestablish(void *arg)
 
 	free(sih, M_DEVBUF, sizeof(*sih));
 }
+
+#ifdef MULTIPROCESSOR
+#include <sys/task.h>
+
+void
+cpu_xcall(struct cpu_info *ci, struct task *t)
+{
+	if (ci == curcpu()) {
+		/* execute the task immediately on the local cpu */
+		int s = splsoftclock();
+		(*t->t_func)(t->t_arg);
+		splx(s);
+	} else {
+		mtx_enter(&ci->ci_xcall_mtx);
+		TAILQ_INSERT_TAIL(&ci->ci_xcall_list, t, t_entry);
+		mtx_leave(&ci->ci_xcall_mtx);
+
+		x86_send_ipi(ci, X86_IPI_XCALL);
+	}
+}
+
+void
+cpu_xcall_dispatch(void)
+{
+	struct cpu_info *ci = curcpu();
+	struct task *list;
+
+	mtx_enter(&ci->ci_xcall_mtx);
+	list = TAILQ_FIRST(&ci->ci_xcall_list);
+	TAILQ_INIT(&ci->ci_xcall_list);
+	mtx_leave(&ci->ci_xcall_mtx);
+
+	while (list != NULL) {
+		struct task *t = list;
+		list = TAILQ_NEXT(t, t_entry);
+
+		(*t->t_func)(t->t_arg);
+	}
+}
+#endif
Index: arch/amd64/amd64/vector.S
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/vector.S,v
retrieving revision 1.47
diff -u -p -r1.47 vector.S
--- arch/amd64/amd64/vector.S	4 Sep 2016 09:22:28 -0000	1.47
+++ arch/amd64/amd64/vector.S	19 Jan 2017 05:32:46 -0000
@@ -1108,3 +1108,11 @@ IDTVEC(softclock)
 	call	_C_LABEL(softintr_dispatch)
 	decl	CPUVAR(IDEPTH)
 	jmp	*%r13
+
+IDTVEC(xcallintr)
+	movl	$IPL_SOFTCLOCK, CPUVAR(ILEVEL)
+	sti
+	incl	CPUVAR(IDEPTH)
+	call	_C_LABEL(cpu_xcall_dispatch)
+	decl	CPUVAR(IDEPTH)
+	jmp	*%r13
Index: arch/amd64/include/cpu.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/cpu.h,v
retrieving revision 1.107
diff -u -p -r1.107 cpu.h
--- arch/amd64/include/cpu.h	14 Dec 2016 10:30:59 -0000	1.107
+++ arch/amd64/include/cpu.h	19 Jan 2017 05:32:46 -0000
@@ -52,6 +52,9 @@
 #include <sys/sched.h>
 #include <sys/sensors.h>
 
+/* for xcalls */
+#include <sys/task.h>
+#include <sys/mutex.h>
 #ifdef _KERNEL
 
 /* VMXON region (Intel) */
@@ -171,6 +174,8 @@ struct cpu_info {
 
 #ifdef MULTIPROCESSOR
 	struct srp_hazard	ci_srp_hazards[SRP_HAZARD_NUM];
+	struct mutex		ci_xcall_mtx;
+	struct task_list	ci_xcall_list;
 #endif
 
 	struct ksensordev	ci_sensordev;
Index: arch/amd64/include/intr.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/intr.h,v
retrieving revision 1.29
diff -u -p -r1.29 intr.h
--- arch/amd64/include/intr.h	13 Sep 2015 11:48:17 -0000	1.29
+++ arch/amd64/include/intr.h	19 Jan 2017 05:32:46 -0000
@@ -219,6 +219,8 @@ void x86_ipi_handler(void);
 void x86_setperf_ipi(struct cpu_info *);
 
 extern void (*ipifunc[X86_NIPI])(struct cpu_info *);
+
+extern void Xxcallintr(void);
 #endif
 
 #endif /* !_LOCORE */
Index: arch/amd64/include/intrdefs.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/intrdefs.h,v
retrieving revision 1.16
diff -u -p -r1.16 intrdefs.h
--- arch/amd64/include/intrdefs.h	22 Jun 2016 01:12:38 -0000	1.16
+++ arch/amd64/include/intrdefs.h	19 Jan 2017 05:32:46 -0000
@@ -53,9 +53,10 @@
 #define	SIR_CLOCK	61
 #define	SIR_NET		60
 #define	SIR_TTY		59
+#define	SIR_XCALL	58
 
-#define	LIR_XEN		58
-#define	LIR_HYPERV	57
+#define	LIR_XEN		57
+#define	LIR_HYPERV	56
 
 /*
  * Maximum # of interrupt sources per CPU. 64 to fit in one word.
@@ -83,13 +84,14 @@
 #define X86_IPI_DDB			0x00000080
 #define X86_IPI_START_VMM		0x00000100
 #define X86_IPI_STOP_VMM		0x00000200
+#define X86_IPI_XCALL			0x00000400
 
-#define X86_NIPI			10
+#define X86_NIPI			11
 
 #define X86_IPI_NAMES { "halt IPI", "nop IPI", "FPU flush IPI", \
 			 "FPU synch IPI", "TLB shootdown IPI", \
 			 "MTRR update IPI", "setperf IPI", "ddb IPI", \
-			 "VMM start IPI", "VMM stop IPI" }
+			 "VMM start IPI", "VMM stop IPI", "xcall IPI" }
 
 #define IREENT_MAGIC	0x18041969
 
Index: sys/systm.h
===================================================================
RCS file: /cvs/src/sys/sys/systm.h,v
retrieving revision 1.121
diff -u -p -r1.121 systm.h
--- sys/systm.h	29 Dec 2016 12:12:44 -0000	1.121
+++ sys/systm.h	19 Jan 2017 05:32:46 -0000
@@ -332,6 +332,10 @@ void	user_config(void);
 #endif
 
 #if defined(MULTIPROCESSOR)
+struct cpu_info;
+struct task;
+void	cpu_xcall(struct cpu_info *, struct task *);
+
 void	_kernel_lock_init(void);
 void	_kernel_lock(void);
 void	_kernel_unlock(void);