--- //depot/user/rwatson/netperf/sys/alpha/alpha/machdep.c	2004/11/06 12:44:50
+++ //depot/user/rwatson/percpu/sys/alpha/alpha/machdep.c	2004/11/18 19:38:17
@@ -2409,3 +2409,29 @@
 	pcpu->pc_idlepcb.apcb_ptbr = thread0.td_pcb->pcb_hw.apcb_ptbr;
 	pcpu->pc_current_asngen = 1;
 }
+
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_ipl = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_ipl);
+	}
+}
--- //depot/user/rwatson/netperf/sys/alpha/alpha/vm_machdep.c	2004/05/31 01:41:33
+++ //depot/user/rwatson/percpu/sys/alpha/alpha/vm_machdep.c	2004/11/18 19:38:17
@@ -203,6 +203,10 @@
 	 */
 	td2->td_md.md_kernnest = 1;
 #endif
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_ipl = ALPHA_PSL_IPL_0;
 }
 
 /*
@@ -322,6 +326,10 @@
 	 */
 	td->td_md.md_kernnest = 1;
 #endif
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_ipl = ALPHA_PSL_IPL_0;
 }
 
 void
--- //depot/user/rwatson/netperf/sys/alpha/include/proc.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/alpha/include/proc.h	2004/11/18 19:38:17
@@ -52,7 +52,8 @@
 	u_int64_t	md_hae;		/* user HAE register value */
 	void            *osf_sigtramp;  /* user-level signal trampoline */
 	u_int		md_kernnest;	/* nesting level in the kernel */
-	register_t	md_savecrit;	/* save PSL for critical section */
+	register_t	md_saved_ipl;	/* save IPL for critical section */
+	u_int		md_spinlock_count;
 };
 
 #define	MDP_UAC_NOPRINT	0x0010		/* Don't print unaligned traps */
--- //depot/user/rwatson/netperf/sys/amd64/amd64/machdep.c	2004/11/03 14:10:39
+++ //depot/user/rwatson/percpu/sys/amd64/amd64/machdep.c	2004/11/18 19:38:17
@@ -1290,6 +1290,32 @@
 	pcpu->pc_acpi_id = 0xffffffff;
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_flags = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_flags);
+	}
+}
+
 /*
  * Construct a PCB from a trapframe. This is called from kdb_trap() where
  * we want to start a backtrace from the function that caused us to enter
--- //depot/user/rwatson/netperf/sys/amd64/amd64/vm_machdep.c	2004/08/18 02:45:50
+++ //depot/user/rwatson/percpu/sys/amd64/amd64/vm_machdep.c	2004/11/18 19:38:17
@@ -143,7 +143,7 @@
 	pcb2->pcb_rsp = (register_t)td2->td_frame - sizeof(void *);
 	pcb2->pcb_rbx = (register_t)td2;		/* fork_trampoline argument */
 	pcb2->pcb_rip = (register_t)fork_trampoline;
-	pcb2->pcb_rflags = td2->td_frame->tf_rflags & ~PSL_I; /* ints disabled */
+	pcb2->pcb_rflags = PSL_KERNEL; /* ints disabled */
 	/*-
 	 * pcb2->pcb_dr*:	cloned above.
 	 * pcb2->pcb_savefpu:	cloned above.
@@ -152,6 +152,10 @@
 	 * pcb2->pcb_[fg]sbase:	cloned above
 	 */
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_flags = pcb2->pcb_rflags | PSL_I;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_rsp is loaded pointing to the cpu_switch() stack frame
@@ -290,6 +294,10 @@
 	 * pcb2->pcb_onfault:	cloned above (always NULL here?).
 	 * pcb2->pcb_[fg]sbase: cloned above
 	 */
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/amd64/include/proc.h	2004/04/07 03:50:45
+++ //depot/user/rwatson/percpu/sys/amd64/include/proc.h	2004/11/18 19:38:17
@@ -37,7 +37,8 @@
  * Machine-dependent part of the proc structure for AMD64.
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_flags;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/arm/arm/machdep.c	2004/11/06 12:44:50
+++ //depot/user/rwatson/percpu/sys/arm/arm/machdep.c	2004/11/18 19:38:17
@@ -315,6 +315,32 @@
 {
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_cspr = disable_interrupts(I32_bit | F32_bit);
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		enable_interrupts(td->td_md.md_saved_cspr);
+	}
+}
+
 /*
  * Clear registers on exec
  */
--- //depot/user/rwatson/netperf/sys/arm/arm/vm_machdep.c	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/arm/arm/vm_machdep.c	2004/11/18 19:38:17
@@ -172,6 +172,10 @@
 	tf->tf_r0 = 0;
 	tf->tf_r1 = 0;
 	pcb2->un_32.pcb32_sp = (u_int)sf;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_cspr = I32_bit | F32_bit;
 }
 				
 void
@@ -306,6 +310,10 @@
 	td->td_pcb->un_32.pcb32_sp = (u_int)sf;
 	td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + td->td_kstack_pages
 	    * PAGE_SIZE - USPACE + USPACE_UNDEF_STACK_TOP;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_cspr = I32_bit | F32_bit;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/arm/include/proc.h	2004/05/23 16:56:02
+++ //depot/user/rwatson/percpu/sys/arm/include/proc.h	2004/11/18 19:38:17
@@ -46,7 +46,8 @@
 };
 
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_cspr;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/conf/files	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/conf/files	2004/11/14 13:39:38
@@ -1673,6 +1673,7 @@
 security/mac_seeotheruids/mac_seeotheruids.c	optional mac_seeotheruids
 security/mac_stub/mac_stub.c	optional mac_stub
 security/mac_test/mac_test.c	optional mac_test
+test/test_synch_timing.c	standard
 ufs/ffs/ffs_alloc.c	optional ffs
 ufs/ffs/ffs_balloc.c	optional ffs
 ufs/ffs/ffs_inode.c	optional ffs
--- //depot/user/rwatson/netperf/sys/conf/files.alpha	2004/08/02 01:36:03
+++ //depot/user/rwatson/percpu/sys/conf/files.alpha	2004/11/18 19:38:17
@@ -43,7 +43,6 @@
 alpha/alpha/clock.c		standard
 alpha/alpha/clock_if.m		standard
 alpha/alpha/cpuconf.c		standard
-alpha/alpha/critical.c		standard
 alpha/alpha/db_disasm.c		optional	ddb
 alpha/alpha/db_interface.c	optional	ddb
 alpha/alpha/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.amd64	2004/10/09 21:40:52
+++ //depot/user/rwatson/percpu/sys/conf/files.amd64	2004/11/18 19:38:17
@@ -59,7 +59,6 @@
 amd64/amd64/bios.c		standard
 amd64/amd64/busdma_machdep.c	standard
 amd64/amd64/cpu_switch.S	standard
-amd64/amd64/critical.c		standard
 amd64/amd64/db_disasm.c		optional	ddb
 amd64/amd64/db_interface.c	optional	ddb
 amd64/amd64/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.arm	2004/05/23 16:56:02
+++ //depot/user/rwatson/percpu/sys/conf/files.arm	2004/11/18 19:38:17
@@ -12,7 +12,6 @@
 arm/arm/cpufunc_asm_sa1.S 	standard
 arm/arm/cpufunc_asm_armv4.S 	standard
 arm/arm/cpufunc_asm_sa11x0.S 	standard
-arm/arm/critical.c		standard
 arm/arm/db_disasm.c		optional	ddb
 arm/arm/db_interface.c		optional	ddb
 arm/arm/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.i386	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/conf/files.i386	2004/11/18 19:38:17
@@ -217,7 +217,6 @@
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard
 i386/i386/busdma_machdep.c	standard
-i386/i386/critical.c		standard
 i386/i386/db_disasm.c		optional	ddb
 i386/i386/db_interface.c	optional	ddb
 i386/i386/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.ia64	2004/10/09 21:40:52
+++ //depot/user/rwatson/percpu/sys/conf/files.ia64	2004/11/18 19:38:17
@@ -91,7 +91,6 @@
 ia64/ia64/clock.c		standard
 ia64/ia64/clock_if.m		standard
 ia64/ia64/context.S		standard
-ia64/ia64/critical.c		standard
 ia64/ia64/db_interface.c	optional	ddb
 ia64/ia64/db_trace.c		optional	ddb
 ia64/ia64/dump_machdep.c	standard
--- //depot/user/rwatson/netperf/sys/conf/files.pc98	2004/09/02 04:10:34
+++ //depot/user/rwatson/percpu/sys/conf/files.pc98	2004/11/18 19:38:17
@@ -141,7 +141,6 @@
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard
 i386/i386/busdma_machdep.c	standard
-i386/i386/critical.c		standard
 i386/i386/db_disasm.c		optional	ddb
 i386/i386/db_interface.c	optional	ddb
 i386/i386/db_trace.c		optional	ddb
--- //depot/user/rwatson/netperf/sys/conf/files.powerpc	2004/08/16 14:53:15
+++ //depot/user/rwatson/percpu/sys/conf/files.powerpc	2004/11/18 19:38:17
@@ -36,7 +36,6 @@
 powerpc/powerpc/copyinout.c	standard
 powerpc/powerpc/copystr.c	standard
 powerpc/powerpc/cpu.c		standard
-powerpc/powerpc/critical.c	standard
 powerpc/powerpc/elf_machdep.c	standard
 powerpc/powerpc/fpu.c		standard
 powerpc/powerpc/fuswintr.c	standard
--- //depot/user/rwatson/netperf/sys/conf/files.sparc64	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/conf/files.sparc64	2004/11/18 19:38:17
@@ -74,7 +74,6 @@
 sparc64/sparc64/cheetah.c	standard
 sparc64/sparc64/clock.c		standard
 sparc64/sparc64/counter.c	standard
-sparc64/sparc64/critical.c	standard
 sparc64/sparc64/db_disasm.c	optional	ddb
 sparc64/sparc64/db_interface.c	optional	ddb
 sparc64/sparc64/db_trace.c	optional	ddb
--- //depot/user/rwatson/netperf/sys/i386/i386/machdep.c	2004/11/03 14:10:39
+++ //depot/user/rwatson/percpu/sys/i386/i386/machdep.c	2004/11/18 19:38:17
@@ -2205,6 +2205,32 @@
 	pcpu->pc_acpi_id = 0xffffffff;
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_flags = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_flags);
+	}
+}
+
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 static void f00f_hack(void *unused);
 SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL)
--- //depot/user/rwatson/netperf/sys/i386/i386/vm_machdep.c	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/i386/i386/vm_machdep.c	2004/11/18 19:38:17
@@ -245,6 +245,10 @@
         }
 	mtx_unlock_spin(&sched_lock);
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_esp is loaded pointing to the cpu_switch() stack frame
@@ -353,7 +357,7 @@
 
 /*
  * Initialize machine state (pcb and trap frame) for a new thread about to
- * upcall. Pu t enough state in the new thread's PCB to get it to go back 
+ * upcall. Put enough state in the new thread's PCB to get it to go back 
  * userret(), where we can intercept it again to set the return (upcall)
  * Address and stack, along with those from upcals that are from other sources
  * such as those generated in thread_userret() itself.
@@ -405,7 +409,7 @@
 	pcb2->pcb_esp = (int)td->td_frame - sizeof(void *); /* trampoline arg */
 	pcb2->pcb_ebx = (int)td;			    /* trampoline arg */
 	pcb2->pcb_eip = (int)fork_trampoline;
-	pcb2->pcb_psl &= ~(PSL_I);	/* interrupts must be disabled */
+	pcb2->pcb_psl = PSL_KERNEL;		/* ints disabled */
 	pcb2->pcb_gs = rgs();
 	/*
 	 * If we didn't copy the pcb, we'd need to do the following registers:
@@ -416,7 +420,11 @@
 	 * pcb2->pcb_gs:	cloned above.  XXXKSE ???
 	 * pcb2->pcb_ext:	cleared below.
 	 */
-	 pcb2->pcb_ext = NULL;
+	pcb2->pcb_ext = NULL;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/i386/include/atomic.h	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/i386/include/atomic.h	2004/11/18 19:38:17
@@ -69,7 +69,7 @@
 
 int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src);
 
-#define	ATOMIC_STORE_LOAD(TYPE, LOP, SOP)			\
+#define	ATOMIC_STORE_LOAD(TYPE, LOP)			\
 u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p);	\
 void		atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 
@@ -191,13 +191,14 @@
 static __inline void					\
 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
 {							\
+	__asm __volatile("" : : : "memory");		\
 	*p = v;						\
 }							\
 struct __hack
 
-#else /* defined(SMP) */
+#else /* !defined(I386_CPU) */
 
-#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP)		\
+#define ATOMIC_STORE_LOAD(TYPE, LOP)			\
 static __inline u_##TYPE				\
 atomic_load_acq_##TYPE(volatile u_##TYPE *p)		\
 {							\
@@ -211,26 +212,21 @@
 	return (res);					\
 }							\
 							\
-/*							\
- * The XCHG instruction asserts LOCK automagically.	\
- */							\
 static __inline void					\
 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
 {							\
-	__asm __volatile(SOP				\
-	: "+m" (*p),			/* 0 */		\
-	  "+r" (v)			/* 1 */		\
-	: : "memory");				 	\
+	__asm __volatile("" : : : "memory");		\
+	*p = v;						\
 }							\
 struct __hack
 
-#endif	/* !defined(SMP) */
+#endif	/* defined(I386_CPU) */
 
 #else /* !(defined(__GNUC__) || defined(__INTEL_COMPILER)) */
 
 extern int atomic_cmpset_int(volatile u_int *, u_int, u_int);
 
-#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP)				\
+#define ATOMIC_STORE_LOAD(TYPE, LOP)					\
 extern u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p);		\
 extern void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 
@@ -258,10 +254,10 @@
 ATOMIC_ASM(add,	     long,  "addl %1,%0",  "ir",  v);
 ATOMIC_ASM(subtract, long,  "subl %1,%0",  "ir",  v);
 
-ATOMIC_STORE_LOAD(char,	"cmpxchgb %b0,%1", "xchgb %b1,%0");
-ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0");
-ATOMIC_STORE_LOAD(int,	"cmpxchgl %0,%1",  "xchgl %1,%0");
-ATOMIC_STORE_LOAD(long,	"cmpxchgl %0,%1",  "xchgl %1,%0");
+ATOMIC_STORE_LOAD(char,	"cmpxchgb %b0,%1");
+ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1");
+ATOMIC_STORE_LOAD(int,	"cmpxchgl %0,%1");
+ATOMIC_STORE_LOAD(long,	"cmpxchgl %0,%1");
 
 #undef ATOMIC_ASM
 #undef ATOMIC_STORE_LOAD
--- //depot/user/rwatson/netperf/sys/i386/include/proc.h	2004/06/28 19:42:50
+++ //depot/user/rwatson/percpu/sys/i386/include/proc.h	2004/11/18 19:38:17
@@ -47,7 +47,8 @@
  * Machine-dependent part of the proc structure for i386.
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_flags;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/ia64/ia64/machdep.c	2004/10/09 21:40:52
+++ //depot/user/rwatson/percpu/sys/ia64/ia64/machdep.c	2004/11/18 19:38:17
@@ -382,6 +382,32 @@
 }
 
 void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_intr = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_intr);
+	}
+}
+
+void
 map_pal_code(void)
 {
 	pt_entry_t pte;
--- //depot/user/rwatson/netperf/sys/ia64/ia64/vm_machdep.c	2004/05/31 01:41:33
+++ //depot/user/rwatson/percpu/sys/ia64/ia64/vm_machdep.c	2004/11/18 19:38:17
@@ -159,6 +159,10 @@
 	pcb->pcb_special.sp = (uintptr_t)tf - 16;
 	pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
 	cpu_set_fork_handler(td, (void (*)(void*))fork_return, td);
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_intr = 1;
 }
 
 void
@@ -272,6 +276,10 @@
 	td2->td_pcb->pcb_special.sp = (uintptr_t)stackp - 16;
 	td2->td_pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
 	cpu_set_fork_handler(td2, (void (*)(void*))fork_return, td2);
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_intr = 1;
 }
 
 /*
--- //depot/user/rwatson/netperf/sys/ia64/include/proc.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/ia64/include/proc.h	2004/11/18 19:38:17
@@ -30,7 +30,8 @@
 #define	_MACHINE_PROC_H_
 
 struct mdthread {
-	register_t	md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_intr;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/kern/kern_fork.c	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/kern/kern_fork.c	2004/11/18 19:38:17
@@ -72,7 +72,6 @@
 #include <vm/uma.h>
 
 #include <sys/user.h>
-#include <machine/critical.h>
 
 #ifndef _SYS_SYSPROTO_H_
 struct fork_args {
@@ -773,9 +772,11 @@
 	td->td_oncpu = PCPU_GET(cpuid);
 	KASSERT(p->p_state == PRS_NORMAL, ("executing process is still new"));
 
+#ifdef __powerpc__
+	cpu_fork_exit(td);
+#endif
 	sched_lock.mtx_lock = (uintptr_t)td;
 	mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED);
-	cpu_critical_fork_exit();
 	CTR4(KTR_PROC, "fork_exit: new thread %p (kse %p, pid %d, %s)",
 		td, td->td_sched, p->p_pid, p->p_comm);
 
--- //depot/user/rwatson/netperf/sys/kern/kern_mutex.c	2004/10/19 18:01:38
+++ //depot/user/rwatson/percpu/sys/kern/kern_mutex.c	2004/11/18 19:38:17
@@ -593,7 +593,7 @@
 			break;
 
 		/* Give interrupts a chance while we spin. */
-		critical_exit();
+		spinlock_exit();
 		while (m->mtx_lock != MTX_UNOWNED) {
 			if (i++ < 10000000) {
 				cpu_spinwait();
@@ -612,7 +612,7 @@
 			}
 			cpu_spinwait();
 		}
-		critical_enter();
+		spinlock_enter();
 	}
 
 	if (LOCK_LOG_TEST(&m->mtx_object, opts))
--- //depot/user/rwatson/netperf/sys/kern/kern_proc.c	2004/10/19 18:01:38
+++ //depot/user/rwatson/percpu/sys/kern/kern_proc.c	2004/11/18 19:38:17
@@ -63,7 +63,6 @@
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/uma.h>
-#include <machine/critical.h>
 
 MALLOC_DEFINE(M_PGRP, "pgrp", "process group header");
 MALLOC_DEFINE(M_SESSION, "session", "session header");
--- //depot/user/rwatson/netperf/sys/kern/kern_switch.c	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/kern/kern_switch.c	2004/11/18 19:38:17
@@ -105,7 +105,6 @@
 #if defined(SMP) && (defined(__i386__) || defined(__amd64__))
 #include <sys/smp.h>
 #endif
-#include <machine/critical.h>
 #if defined(SMP) && defined(SCHED_4BSD)
 #include <sys/sysctl.h>
 #endif
@@ -564,18 +563,17 @@
  * Kernel thread preemption implementation.  Critical sections mark
  * regions of code in which preemptions are not allowed.
  */
+#ifdef INVARIANT_SUPPORT
 void
-critical_enter(void)
+_critical_assert(const char *file, int line)
 {
-	struct thread *td;
 
-	td = curthread;
-	if (td->td_critnest == 0)
-		cpu_critical_enter(td);
-	td->td_critnest++;
-	CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td,
-	    (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest);
+	if (curthread->td_critnest == 0) {
+		panic("critical_asssert: critical section not owned "
+		    "at %s:%d", file, line);
+	}
 }
+#endif
 
 void
 critical_exit(void)
@@ -599,7 +597,6 @@
 		}
 #endif
 		td->td_critnest = 0;
-		cpu_critical_exit(td);
 	} else {
 		td->td_critnest--;
 	}
--- //depot/user/rwatson/netperf/sys/powerpc/include/cpufunc.h	2004/08/07 02:40:42
+++ //depot/user/rwatson/percpu/sys/powerpc/include/cpufunc.h	2004/11/18 19:38:17
@@ -170,6 +170,14 @@
 	return(ret);
 }
 
+/* XXXFIXME: This needs to go away. */
+static __inline void
+cpu_fork_exit(struct thread *td)
+{
+
+	td->td_md.md_saved_msr = (mfmsr() | PSL_EE | PSL_RI);
+}
+
 #endif /* _KERNEL */
 
 #endif /* !_MACHINE_CPUFUNC_H_ */
--- //depot/user/rwatson/netperf/sys/powerpc/include/proc.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/powerpc/include/proc.h	2004/11/18 19:38:17
@@ -39,7 +39,8 @@
  * Machine-dependent part of the proc structure
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_msr;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/powerpc/powerpc/machdep.c	2004/09/05 16:49:28
+++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/machdep.c	2004/11/18 19:38:17
@@ -878,6 +878,32 @@
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_msr = intr_disable();
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		intr_restore(td->td_md.md_saved_msr);
+	}
+}
+
 /*
  * kcopy(const void *src, void *dst, size_t len);
  *
--- //depot/user/rwatson/netperf/sys/powerpc/powerpc/swtch.S	2004/07/24 17:51:01
+++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/swtch.S	2004/11/18 19:38:17
@@ -143,6 +143,14 @@
  * Set up the return from cpu_fork()
  */
 ENTRY(fork_trampoline)
+#if 0
+	/* XXX:	 Are r2, r6, and r7 ok to use here? */
+	mfmsr	%r2			/* Setup td_md.md_saved_msr. */
+	ori	%r2, %r2, PSL_EE | PSL_RI
+	mfsprg	%r7,0
+	lwz	%r6,PC_CURTHREAD(%r7)	/* Load curthread. */
+	stw	%r2,TD_MD_SAVED_MSR(%r6)
+#endif
 	lwz	%r3,CF_FUNC(%r1)
 	lwz	%r4,CF_ARG0(%r1)
 	lwz	%r5,CF_ARG1(%r1)
--- //depot/user/rwatson/netperf/sys/powerpc/powerpc/vm_machdep.c	2004/07/24 17:51:01
+++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/vm_machdep.c	2004/11/18 19:38:17
@@ -155,6 +155,12 @@
 	pcb->pcb_lr = (register_t)fork_trampoline;
 	pcb->pcb_usr = kernel_pmap->pm_sr[USER_SR];
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+#if 0
+	td2->td_md.md_saved_msr = XXX /* something like PSL_KERNEL? */;
+#endif
+
 	/*
  	 * Now cpu_switch() can schedule the new process.
 	 */
@@ -330,6 +336,12 @@
 	pcb2->pcb_sp = (register_t)cf;
 	pcb2->pcb_lr = (register_t)fork_trampoline;
 	pcb2->pcb_usr = kernel_pmap->pm_sr[USER_SR];
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+#if 0
+	td->td_md.md_saved_msr = XXX /* something like PSL_KERNEL? */;
+#endif
 }
 
 void
--- //depot/user/rwatson/netperf/sys/sparc64/include/proc.h	2004/04/08 03:11:34
+++ //depot/user/rwatson/percpu/sys/sparc64/include/proc.h	2004/11/18 19:38:17
@@ -42,7 +42,8 @@
 };
 
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_pil;	/* (k) */
 };
 
 struct mdproc {
--- //depot/user/rwatson/netperf/sys/sparc64/sparc64/machdep.c	2004/10/09 21:40:52
+++ //depot/user/rwatson/percpu/sys/sparc64/sparc64/machdep.c	2004/11/18 19:38:17
@@ -232,6 +232,33 @@
 	}
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_pil = rdpr(pil);
+		wrpr(pil, 0, 14);
+		critical_enter();
+	}
+	td->td_md.md_spinlock_count++;
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0) {
+		critical_exit();
+		wrpr(pil, td->td_md.md_saved_pil, 0);
+	}
+}
+
 unsigned
 tick_get_timecount(struct timecounter *tc)
 {
--- //depot/user/rwatson/netperf/sys/sparc64/sparc64/vm_machdep.c	2004/11/14 11:53:24
+++ //depot/user/rwatson/percpu/sys/sparc64/sparc64/vm_machdep.c	2004/11/18 19:38:17
@@ -173,6 +173,10 @@
 	fr->fr_local[2] = (u_long)tf;
 	pcb->pcb_pc = (u_long)fork_trampoline - 8;
 	pcb->pcb_sp = (u_long)fr - SPOFF;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_pil = 0;
 }
 
 void
@@ -287,6 +291,10 @@
 	pcb2->pcb_sp = (u_long)fp - SPOFF;
 	pcb2->pcb_pc = (u_long)fork_trampoline - 8;
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_pil = 0;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 */
--- //depot/user/rwatson/netperf/sys/sys/lock.h	2004/02/28 22:29:37
+++ //depot/user/rwatson/percpu/sys/sys/lock.h	2004/11/18 19:38:17
@@ -196,6 +196,8 @@
 extern struct lock_class lock_class_mtx_spin;
 extern struct lock_class lock_class_sx;
 
+void	spinlock_enter(void);
+void	spinlock_exit(void);
 void	witness_init(struct lock_object *);
 void	witness_destroy(struct lock_object *);
 int	witness_defineorder(struct lock_object *, struct lock_object *);
--- //depot/user/rwatson/netperf/sys/sys/mutex.h	2004/08/05 21:29:14
+++ //depot/user/rwatson/percpu/sys/sys/mutex.h	2004/11/18 19:38:17
@@ -164,7 +164,7 @@
 #define _get_spin_lock(mp, tid, opts, file, line) do {			\
 	struct thread *_tid = (tid);					\
 									\
-	critical_enter();						\
+	spinlock_enter();						\
 	if (!_obtain_lock((mp), _tid)) {				\
 		if ((mp)->mtx_lock == (uintptr_t)_tid)			\
 			(mp)->mtx_recurse++;				\
@@ -191,8 +191,8 @@
  * Since spin locks are not _too_ common, inlining this code is not too big 
  * a deal.
  *
- * Since we always perform a critical_enter() when attempting to acquire a
- * spin lock, we need to always perform a matching critical_exit() when
+ * Since we always perform a spinlock_enter() when attempting to acquire a
+ * spin lock, we need to always perform a matching spinlock_exit() when
  * releasing a spin lock.  This includes the recursion cases.
  */
 #ifndef _rel_spin_lock
@@ -201,7 +201,7 @@
 		(mp)->mtx_recurse--;					\
 	else								\
 		_release_lock_quick((mp));				\
-	critical_exit();						\
+	spinlock_exit();						\
 } while (0)
 #endif
 
--- //depot/user/rwatson/netperf/sys/sys/systm.h	2004/11/03 14:10:39
+++ //depot/user/rwatson/percpu/sys/sys/systm.h	2004/11/18 19:38:17
@@ -86,6 +86,12 @@
 #define	__CTASSERT(x, y)	typedef char __assert ## y[(x) ? 1 : -1]
 #endif
 
+#ifdef	INVARIANTS
+#define	critical_assert()	_critical_assert(__FILE__, __LINE__)
+#else
+#define	critical_assert()
+#endif
+
 /*
  * XXX the hints declarations are even more misplaced than most declarations
  * in this file, since they are needed in one file (per arch) and only used
@@ -139,7 +145,10 @@
 void	cpu_rootconf(void);
 extern uint32_t crc32_tab[];
 uint32_t crc32(const void *buf, size_t size);
-void	critical_enter(void);
+#ifdef INVARIANT_SUPPORT
+void	_critical_assert(const char *file, int line);
+#endif
+#define	critical_enter()	curthread->td_critnest++
 void	critical_exit(void);
 void	init_param1(void);
 void	init_param2(long physpages);
--- //depot/user/rwatson/netperf/sys/vm/uma_core.c	2004/11/06 12:44:50
+++ //depot/user/rwatson/percpu/sys/vm/uma_core.c	2004/11/13 09:42:54
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2004, Robert N. M. Watson
  * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
  * All rights reserved.
  *
@@ -382,48 +383,19 @@
 zone_timeout(uma_zone_t zone)
 {
 	uma_keg_t keg;
-	uma_cache_t cache;
 	u_int64_t alloc;
-	int cpu;
 
 	keg = zone->uz_keg;
 	alloc = 0;
 
 	/*
-	 * Aggregate per cpu cache statistics back to the zone.
-	 *
-	 * XXX This should be done in the sysctl handler.
-	 *
-	 * I may rewrite this to set a flag in the per cpu cache instead of
-	 * locking.  If the flag is not cleared on the next round I will have
-	 * to lock and do it here instead so that the statistics don't get too
-	 * far out of sync.
-	 */
-	if (!(keg->uk_flags & UMA_ZFLAG_INTERNAL)) {
-		for (cpu = 0; cpu <= mp_maxid; cpu++) {
-			if (CPU_ABSENT(cpu))
-				continue;
-			CPU_LOCK(cpu);
-			cache = &zone->uz_cpu[cpu];
-			/* Add them up, and reset */
-			alloc += cache->uc_allocs;
-			cache->uc_allocs = 0;
-			CPU_UNLOCK(cpu);
-		}
-	}
-
-	/* Now push these stats back into the zone.. */
-	ZONE_LOCK(zone);
-	zone->uz_allocs += alloc;
-
-	/*
 	 * Expand the zone hash table.
 	 *
 	 * This is done if the number of slabs is larger than the hash size.
 	 * What I'm trying to do here is completely reduce collisions.  This
 	 * may be a little aggressive.  Should I allow for two collisions max?
 	 */
-
+	ZONE_LOCK(zone);
 	if (keg->uk_flags & UMA_ZONE_HASH &&
 	    keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
 		struct uma_hash newhash;
@@ -611,6 +583,10 @@
 /*
  * Drains the per cpu caches for a zone.
  *
+ * NOTE: This may only be called while the zone is being turn down, and not
+ * during normal operation.  This is necessary in order that we do not have
+ * to migrate CPUs to drain the per-CPU caches.
+ *
  * Arguments:
  *	zone     The zone to drain, must be unlocked.
  *
@@ -624,12 +600,20 @@
 	int cpu;
 
 	/*
-	 * We have to lock each cpu cache before locking the zone
+	 * XXX: It is safe to not lock the per-CPU caches, because we're
+	 * tearing down the zone anyway.  I.e., there will be no further use
+	 * of the caches at this point.
+	 *
+	 * XXX: It would good to be able to assert that the zone is being
+	 * torn down to prevent improper use of cache_drain().
+	 *
+	 * XXX: We lock the zone before passing into bucket_cache_drain() as
+	 * it is used elsewhere.  Should the tear-down path be made special
+	 * there in some form?
 	 */
 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
 		if (CPU_ABSENT(cpu))
 			continue;
-		CPU_LOCK(cpu);
 		cache = &zone->uz_cpu[cpu];
 		bucket_drain(zone, cache->uc_allocbucket);
 		bucket_drain(zone, cache->uc_freebucket);
@@ -642,11 +626,6 @@
 	ZONE_LOCK(zone);
 	bucket_cache_drain(zone);
 	ZONE_UNLOCK(zone);
-	for (cpu = 0; cpu <= mp_maxid; cpu++) {
-		if (CPU_ABSENT(cpu))
-			continue;
-		CPU_UNLOCK(cpu);
-	}
 }
 
 /*
@@ -1784,6 +1763,7 @@
 	uma_cache_t cache;
 	uma_bucket_t bucket;
 	int cpu;
+	int count;
 	int badness;
 
 	/* This is the fast path allocation */
@@ -1818,12 +1798,30 @@
 		}
 	}
 
+	/*
+	 * If possible, allocate from the per-CPU cache.  There are two
+	 * requirements for safe access to the per-CPU cache: first, we must
+	 * not be preempted or yield during access, and second, we must not
+	 * migrate CPUs without swiching the cache we are accessing.  In the
+	 * current implementation, we rely on a critical section to prevent
+	 * preemption and migration.  We do have to release the critical
+	 * section when acquiring the zone mutex if there's a cache miss, in
+	 * which case we must make sure the cache is in a consistent state
+	 * before doing so, and we must detect migration if it occurs and
+	 * change which cache we are accessing one the critical section is
+	 * re-acquired.  This is cheaper than unconditionally pinning, as the
+	 * vast majority of operations hit in the cache.
+	 */
+	count = 0;
 zalloc_restart:
+	critical_enter();
 	cpu = PCPU_GET(cpuid);
-	CPU_LOCK(cpu);
 	cache = &zone->uz_cpu[cpu];
 
 zalloc_start:
+	count++;
+	KASSERT(count < 10, ("uma_zalloc_arg: count == 10"));
+	critical_assert();
 	bucket = cache->uc_allocbucket;
 
 	if (bucket) {
@@ -1836,12 +1834,12 @@
 			KASSERT(item != NULL,
 			    ("uma_zalloc: Bucket pointer mangled."));
 			cache->uc_allocs++;
+			critical_exit();
 #ifdef INVARIANTS
 			ZONE_LOCK(zone);
 			uma_dbg_alloc(zone, NULL, item);
 			ZONE_UNLOCK(zone);
 #endif
-			CPU_UNLOCK(cpu);
 			if (zone->uz_ctor != NULL) {
 				if (zone->uz_ctor(item, zone->uz_keg->uk_size,
 				    udata, flags) != 0) {
@@ -1871,7 +1869,33 @@
 			}
 		}
 	}
+	/*
+	 * Attempt to retrieve the item from the per-CPU cache has failed,
+	 * so we must go back to the zone.  This requires the zone lock, so
+	 * we must drop the critical section, then re-acquire it when we go
+	 * back to the cache.  Since the critical section is released, we
+	 * may be preempted or migrate.  As such, make sure not to maintain
+	 * any thread-local state specific to the cache from prior to
+	 * releasing the critical section.
+	 */
+	critical_exit();
 	ZONE_LOCK(zone);
+	critical_enter();
+	cpu = PCPU_GET(cpuid);
+	cache = &zone->uz_cpu[cpu];
+	bucket = cache->uc_allocbucket;
+	if (bucket != NULL) {
+		if (bucket != NULL && bucket->ub_cnt > 0) {
+			ZONE_UNLOCK(zone);
+			goto zalloc_start;
+		}
+		bucket = cache->uc_freebucket;
+		if (bucket != NULL && bucket->ub_cnt > 0) {
+			ZONE_UNLOCK(zone);
+			goto zalloc_start;
+		}
+	}
+
 	/* Since we have locked the zone we may as well send back our stats */
 	zone->uz_allocs += cache->uc_allocs;
 	cache->uc_allocs = 0;
@@ -1896,7 +1920,7 @@
 		goto zalloc_start;
 	}
 	/* We are no longer associated with this cpu!!! */
-	CPU_UNLOCK(cpu);
+	critical_exit();
 
 	/* Bump up our uz_count so we get here less */
 	if (zone->uz_count < BUCKET_MAX)
@@ -2209,6 +2233,7 @@
 	uma_bucket_t bucket;
 	int bflags;
 	int cpu;
+	int count;
 	enum zfreeskip skip;
 
 	/* This is the fast path free */
@@ -2234,12 +2259,16 @@
 		skip = SKIP_DTOR;
 	}
 
+	count = 0;
 zfree_restart:
+	critical_enter();
 	cpu = PCPU_GET(cpuid);
-	CPU_LOCK(cpu);
 	cache = &zone->uz_cpu[cpu];
 
 zfree_start:
+	count++;
+	KASSERT(count < 10, ("uma_zfree_arg: count == 10"));
+	critical_assert();
 	bucket = cache->uc_freebucket;
 
 	if (bucket) {
@@ -2253,6 +2282,7 @@
 			    ("uma_zfree: Freeing to non free bucket index."));
 			bucket->ub_bucket[bucket->ub_cnt] = item;
 			bucket->ub_cnt++;
+			critical_exit();
 #ifdef INVARIANTS
 			ZONE_LOCK(zone);
 			if (keg->uk_flags & UMA_ZONE_MALLOC)
@@ -2261,7 +2291,6 @@
 				uma_dbg_free(zone, NULL, item);
 			ZONE_UNLOCK(zone);
 #endif
-			CPU_UNLOCK(cpu);
 			return;
 		} else if (cache->uc_allocbucket) {
 #ifdef UMA_DEBUG_ALLOC
@@ -2285,9 +2314,33 @@
 	 *
 	 * 1) The buckets are NULL
 	 * 2) The alloc and free buckets are both somewhat full.
+	 *
+	 * Since we acquire the zone lock, we must first release the
+	 * critical section, as we might otherwise sleep.  As a result, we
+	 * may be preempted or migrate, so be careful not to maintain any
+	 * potentially stale thread-local state regarding the cache,
+	 * reloading it all once the critical section is re-acquired.  If
+	 * we have migrated, we may not even need to do any work, so loop
+	 * if the conditions have changed.
 	 */
-
+	critical_exit();
 	ZONE_LOCK(zone);
+	critical_enter();
+	cpu = PCPU_GET(cpuid);
+	cache = &zone->uz_cpu[cpu];
+	if (cache->uc_freebucket != NULL) {
+		if (cache->uc_freebucket->ub_cnt <
+		    cache->uc_freebucket->ub_entries) {
+			ZONE_UNLOCK(zone);
+			goto zfree_start;
+		}
+		if (cache->uc_allocbucket != NULL &&
+		    (cache->uc_allocbucket->ub_cnt <
+		    cache->uc_freebucket->ub_cnt)) {
+			ZONE_UNLOCK(zone);
+			goto zfree_start;
+		}
+	}
 
 	bucket = cache->uc_freebucket;
 	cache->uc_freebucket = NULL;
@@ -2310,7 +2363,7 @@
 		goto zfree_start;
 	}
 	/* We're done with this CPU now */
-	CPU_UNLOCK(cpu);
+	critical_exit();
 
 	/* And the zone.. */
 	ZONE_UNLOCK(zone);
@@ -2724,6 +2777,7 @@
 	int cachefree;
 	uma_bucket_t bucket;
 	uma_cache_t cache;
+	u_int64_t alloc;
 
 	cnt = 0;
 	mtx_lock(&uma_mtx);
@@ -2747,6 +2801,13 @@
 	  LIST_FOREACH(z, &zk->uk_zones, uz_link) {
 		if (cnt == 0)	/* list may have changed size */
 			break;
+		/*
+		 * XXXRW: This is the last remaining use of a per-CPU mutex
+		 * to protect access to a per-CPU zone cache.  We should be
+		 * migrating CPUs here to pick up data from each cache
+		 * instead.  Because this locking no longer protects other
+		 * use of the cache, we can now get inconsistent results.
+		 */
 		if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) {
 			for (cpu = 0; cpu <= mp_maxid; cpu++) {
 				if (CPU_ABSENT(cpu))
@@ -2756,6 +2817,7 @@
 		}
 		ZONE_LOCK(z);
 		cachefree = 0;
+		alloc = 0;
 		if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) {
 			for (cpu = 0; cpu <= mp_maxid; cpu++) {
 				if (CPU_ABSENT(cpu))
@@ -2765,9 +2827,15 @@
 					cachefree += cache->uc_allocbucket->ub_cnt;
 				if (cache->uc_freebucket != NULL)
 					cachefree += cache->uc_freebucket->ub_cnt;
+				alloc += cache->uc_allocs;
+				cache->uc_allocs = 0;
 				CPU_UNLOCK(cpu);
 			}
 		}
+
+		/* Now push alloc stats back into zone.  */
+		z->uz_allocs += alloc;
+
 		LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) {
 			cachefree += bucket->ub_cnt;
 		}