Index: amd64/acpi_machdep.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/acpi_machdep.c,v retrieving revision 1.67 diff -u -p -r1.67 acpi_machdep.c --- amd64/acpi_machdep.c 18 Dec 2014 05:33:48 -0000 1.67 +++ amd64/acpi_machdep.c 5 Jan 2015 00:38:36 -0000 @@ -425,7 +425,7 @@ acpi_resume_mp(void) pcb->pcb_rsp = (u_int64_t)sf; pcb->pcb_rbp = 0; - ci->ci_idepth = 0; + ci->ci_ifloor = IPL_NONE; ci->ci_flags &= ~CPUF_PRESENT; cpu_start_secondary(ci); Index: amd64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/cpu.c,v retrieving revision 1.76 diff -u -p -r1.76 cpu.c --- amd64/cpu.c 18 Dec 2014 16:23:26 -0000 1.76 +++ amd64/cpu.c 5 Jan 2015 00:38:36 -0000 @@ -432,6 +432,7 @@ cpu_attach(struct device *parent, struct ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */ #endif ci->ci_func = caa->cpu_func; + ci->ci_ifloor = IPL_NONE; #if defined(MULTIPROCESSOR) /* Index: amd64/genassym.cf =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/genassym.cf,v retrieving revision 1.30 diff -u -p -r1.30 genassym.cf --- amd64/genassym.cf 1 Apr 2014 09:05:03 -0000 1.30 +++ amd64/genassym.cf 5 Jan 2015 00:38:36 -0000 @@ -105,7 +105,6 @@ member CPU_INFO_CURPROC ci_curproc member CPU_INFO_CURPCB ci_curpcb member CPU_INFO_IDLE_PCB ci_idle_pcb member CPU_INFO_ILEVEL ci_ilevel -member CPU_INFO_IDEPTH ci_idepth member CPU_INFO_ISOURCES ci_isources member CPU_INFO_IPENDING ci_ipending member CPU_INFO_IUNMASK ci_iunmask Index: amd64/intr.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/intr.c,v retrieving revision 1.39 diff -u -p -r1.39 intr.c --- amd64/intr.c 2 Dec 2014 18:13:10 -0000 1.39 +++ amd64/intr.c 5 Jan 2015 00:38:36 -0000 @@ -514,6 +514,8 @@ intr_disestablish(struct intrhand *ih) int intr_handler(struct intrframe *frame, struct intrhand *ih) { + struct cpu_info *ci = curcpu(); + int floor; int rc; #ifdef MULTIPROCESSOR int need_lock; @@ -526,7 +528,10 @@ intr_handler(struct intrframe *frame, st if (need_lock) __mp_lock(&kernel_lock); #endif + floor = ci->ci_ifloor; + ci->ci_ifloor = ih->ih_level; rc = (*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : frame); + ci->ci_ifloor = floor; #ifdef MULTIPROCESSOR if (need_lock) __mp_unlock(&kernel_lock); Index: amd64/ipi.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/ipi.c,v retrieving revision 1.11 diff -u -p -r1.11 ipi.c --- amd64/ipi.c 5 Dec 2012 23:20:10 -0000 1.11 +++ amd64/ipi.c 5 Jan 2015 00:38:36 -0000 @@ -102,8 +102,10 @@ x86_ipi_handler(void) struct cpu_info *ci = curcpu(); u_int32_t pending; int bit; + int floor; - pending = x86_atomic_testset_u32(&ci->ci_ipis, 0); + pending = atomic_swap_uint(&ci->ci_ipis, 0); + floor = atomic_swap_uint(&ci->ci_ifloor, ci->ci_ilevel); for (bit = 0; bit < X86_NIPI && pending; bit++) { if (pending & (1<ci_ifloor = floor; } Index: amd64/lapic.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/lapic.c,v retrieving revision 1.36 diff -u -p -r1.36 lapic.c --- amd64/lapic.c 18 Dec 2014 05:33:48 -0000 1.36 +++ amd64/lapic.c 5 Jan 2015 00:38:36 -0000 @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -272,8 +273,12 @@ u_int32_t lapic_delaytab[26]; void lapic_clockintr(void *arg, struct intrframe frame) { + struct cpu_info *ci = curcpu(); + int floor; + floor = atomic_swap_uint(&ci->ci_ifloor, ci->ci_ilevel); hardclock((struct clockframe *)&frame); + ci->ci_ifloor = floor; clk_count.ec_count++; } Index: amd64/machdep.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/machdep.c,v retrieving revision 1.204 diff -u -p -r1.204 machdep.c --- amd64/machdep.c 21 Dec 2014 16:27:07 -0000 1.204 +++ amd64/machdep.c 5 Jan 2015 00:38:36 -0000 @@ -1774,14 +1774,15 @@ void splassert_check(int wantipl, const char *func) { int cpl = curcpu()->ci_ilevel; + int floor = curcpu()->ci_ifloor; if (cpl < wantipl) { splassert_fail(wantipl, cpl, func); } - - if (wantipl == IPL_NONE && curcpu()->ci_idepth != 0) { - splassert_fail(-1, curcpu()->ci_idepth, func); + if (floor > wantipl) { + splassert_fail(wantipl, floor, func); } + } #endif Index: amd64/pmap.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/pmap.c,v retrieving revision 1.83 diff -u -p -r1.83 pmap.c --- amd64/pmap.c 23 Dec 2014 07:42:46 -0000 1.83 +++ amd64/pmap.c 5 Jan 2015 00:38:36 -0000 @@ -216,7 +216,8 @@ pd_entry_t *normal_pdes[] = PDES_INITIAL #define COUNT(x) /* nothing */ -#define pmap_pte_set(p, n) x86_atomic_testset_u64(p, n) +#define pmap_pte_set(p, n) \ + (u_int64_t)atomic_swap_ulong((unsigned long *)(p), (unsigned long)(n)) #define pmap_pte_clearbits(p, b) x86_atomic_clearbits_u64(p, b) #define pmap_pte_setbits(p, b) x86_atomic_setbits_u64(p, b) Index: amd64/softintr.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/softintr.c,v retrieving revision 1.6 diff -u -p -r1.6 softintr.c --- amd64/softintr.c 12 Jul 2014 18:44:41 -0000 1.6 +++ amd64/softintr.c 5 Jan 2015 00:38:36 -0000 @@ -76,8 +76,12 @@ softintr_init(void) void softintr_dispatch(int which) { + struct cpu_info *ci = curcpu(); struct x86_soft_intr *si = &x86_soft_intrs[which]; struct x86_soft_intrhand *sih; + int floor; + + floor = atomic_swap_uint(&ci->ci_ifloor, ci->ci_ilevel); for (;;) { mtx_enter(&si->softintr_lock); @@ -95,6 +99,8 @@ softintr_dispatch(int which) (*sih->sih_fn)(sih->sih_arg); } + + ci->ci_ifloor = floor; } /* Index: amd64/spl.S =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/spl.S,v retrieving revision 1.8 diff -u -p -r1.8 spl.S --- amd64/spl.S 16 Jun 2011 19:46:39 -0000 1.8 +++ amd64/spl.S 5 Jan 2015 00:38:36 -0000 @@ -147,7 +147,6 @@ IDTVEC(spllower) */ IDTVEC(doreti) popq %rbx # get previous priority - decl CPUVAR(IDEPTH) leaq 1f(%rip),%r13 1: movl %ebx,%eax movq CPUVAR(IUNMASK)(,%rax,8),%rax Index: amd64/vector.S =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/vector.S,v retrieving revision 1.35 diff -u -p -r1.35 vector.S --- amd64/vector.S 23 Nov 2014 00:25:05 -0000 1.35 +++ amd64/vector.S 5 Jan 2015 00:38:36 -0000 @@ -331,7 +331,6 @@ IDTVEC(intr_lapic_ipi) jae 2f IDTVEC(resume_lapic_ipi) 1: - incl CPUVAR(IDEPTH) movl $IPL_IPI,CPUVAR(ILEVEL) sti cld @@ -413,7 +412,6 @@ IDTVEC(intr_lapic_ltimer) jae 2f IDTVEC(resume_lapic_ltimer) 1: - incl CPUVAR(IDEPTH) movl $IPL_CLOCK,CPUVAR(ILEVEL) sti cld @@ -468,7 +466,6 @@ IDTVEC(intr_##name##num) ;\ movl %ebx,CPUVAR(ILEVEL) ;\ sti ;\ cld ;\ - incl CPUVAR(IDEPTH) ;\ movq IS_HANDLERS(%r14),%rbx ;\ 6: \ movl IH_LEVEL(%rbx),%r12d ;\ @@ -995,7 +992,6 @@ _C_LABEL(ioapic_level_stubs): IDTVEC(softtty) movl $IPL_SOFTTTY, CPUVAR(ILEVEL) sti - incl CPUVAR(IDEPTH) #ifdef MULTIPROCESSOR call _C_LABEL(x86_softintlock) #endif @@ -1005,13 +1001,11 @@ IDTVEC(softtty) #ifdef MULTIPROCESSOR call _C_LABEL(x86_softintunlock) #endif - decl CPUVAR(IDEPTH) jmp *%r13 IDTVEC(softnet) movl $IPL_SOFTNET, CPUVAR(ILEVEL) sti - incl CPUVAR(IDEPTH) #ifdef MULTIPROCESSOR call _C_LABEL(x86_softintlock) #endif @@ -1021,13 +1015,11 @@ IDTVEC(softnet) #ifdef MULTIPROCESSOR call _C_LABEL(x86_softintunlock) #endif - decl CPUVAR(IDEPTH) jmp *%r13 IDTVEC(softclock) movl $IPL_SOFTCLOCK, CPUVAR(ILEVEL) sti - incl CPUVAR(IDEPTH) #ifdef MULTIPROCESSOR call _C_LABEL(x86_softintlock) #endif @@ -1038,5 +1030,4 @@ IDTVEC(softclock) #ifdef MULTIPROCESSOR call _C_LABEL(x86_softintunlock) #endif - decl CPUVAR(IDEPTH) jmp *%r13 Index: include/atomic.h =================================================================== RCS file: /cvs/src/sys/arch/amd64/include/atomic.h,v retrieving revision 1.16 diff -u -p -r1.16 atomic.h --- include/atomic.h 8 Oct 2014 19:40:28 -0000 1.16 +++ include/atomic.h 5 Jan 2015 00:38:36 -0000 @@ -88,6 +88,39 @@ _atomic_cas_ptr(volatile void *p, void * } #define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n)) +static inline unsigned int +_atomic_swap_uint(volatile unsigned int *p, unsigned int n) +{ + __asm volatile("xchgl %0, %1" + : "=a" (n), "=m" (*p) + : "0" (n), "m" (*p)); + + return (n); +} +#define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n)) + +static inline unsigned long +_atomic_swap_ulong(volatile unsigned long *p, unsigned long n) +{ + __asm volatile("xchgq %0, %1" + : "=a" (n), "=m" (*p) + : "0" (n), "m" (*p)); + + return (n); +} +#define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n)) + +static inline void * +_atomic_swap_ptr(volatile void *p, void *n) +{ + __asm volatile("xchgq %0, %1" + : "=a" (n), "=m" (*(unsigned long *)p) + : "0" (n), "m" (*(unsigned long *)p)); + + return (n); +} +#define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n)) + static inline void _atomic_inc_int(volatile unsigned int *p) { @@ -235,21 +268,6 @@ _atomic_sub_long_nv(volatile unsigned lo #define virtio_membar_producer() __membar("") #define virtio_membar_consumer() __membar("") #define virtio_membar_sync() __membar("mfence") - -static __inline u_int64_t -x86_atomic_testset_u64(volatile u_int64_t *ptr, u_int64_t val) -{ - __asm__ volatile ("xchgq %0,(%2)" :"=r" (val):"0" (val),"r" (ptr)); - return val; -} - -static __inline u_int32_t -x86_atomic_testset_u32(volatile u_int32_t *ptr, u_int32_t val) -{ - __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr)); - return val; -} - static __inline void x86_atomic_setbits_u32(volatile u_int32_t *ptr, u_int32_t bits) Index: include/cpu.h =================================================================== RCS file: /cvs/src/sys/arch/amd64/include/cpu.h,v retrieving revision 1.88 diff -u -p -r1.88 cpu.h --- include/cpu.h 16 Dec 2014 21:20:23 -0000 1.88 +++ include/cpu.h 5 Jan 2015 00:38:36 -0000 @@ -84,7 +84,7 @@ struct cpu_info { struct intrsource *ci_isources[MAX_INTR_SOURCES]; u_int64_t ci_ipending; int ci_ilevel; - int ci_idepth; + int ci_ifloor; u_int64_t ci_imask[NIPL]; u_int64_t ci_iunmask[NIPL]; #ifdef DIAGNOSTIC @@ -247,7 +247,7 @@ extern struct cpu_info cpu_info_primary; #define CLKF_USERMODE(frame) USERMODE((frame)->if_cs, (frame)->if_rflags) #define CLKF_PC(frame) ((frame)->if_rip) -#define CLKF_INTR(frame) (curcpu()->ci_idepth > 1) +#define CLKF_INTR(frame) (curcpu()->ci_ipending != 0) /* * This is used during profiling to integrate system time.