--- //depot/user/rwatson/netperf/sys/alpha/alpha/machdep.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/alpha/alpha/machdep.c 2005/01/19 20:22:17 @@ -2406,3 +2406,29 @@ pcpu->pc_idlepcb.apcb_ptbr = thread0.td_pcb->pcb_hw.apcb_ptbr; pcpu->pc_current_asngen = 1; } + +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_ipl = intr_disable(); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + intr_restore(td->td_md.md_saved_ipl); + } +} --- //depot/user/rwatson/netperf/sys/alpha/alpha/vm_machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/alpha/alpha/vm_machdep.c 2005/01/19 20:22:17 @@ -202,6 +202,10 @@ */ td2->td_md.md_kernnest = 1; #endif + + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_ipl = ALPHA_PSL_IPL_0; } /* @@ -321,6 +325,10 @@ */ td->td_md.md_kernnest = 1; #endif + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_ipl = ALPHA_PSL_IPL_0; } void --- //depot/user/rwatson/netperf/sys/alpha/include/proc.h 2004/02/28 22:29:37 +++ //depot/user/rwatson/percpu/sys/alpha/include/proc.h 2004/11/25 02:37:15 @@ -52,7 +52,8 @@ u_int64_t md_hae; /* user HAE register value */ void *osf_sigtramp; /* user-level signal trampoline */ u_int md_kernnest; /* nesting level in the kernel */ - register_t md_savecrit; /* save PSL for critical section */ + register_t md_saved_ipl; /* save IPL for critical section */ + u_int md_spinlock_count; }; #define MDP_UAC_NOPRINT 0x0010 /* Don't print unaligned traps */ --- //depot/user/rwatson/netperf/sys/amd64/amd64/machdep.c 2004/12/04 17:43:45 +++ //depot/user/rwatson/percpu/sys/amd64/amd64/machdep.c 2005/01/19 20:22:17 @@ -1287,6 +1287,32 @@ pcpu->pc_acpi_id = 0xffffffff; } +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_flags = intr_disable(); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + intr_restore(td->td_md.md_saved_flags); + } +} + /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter --- //depot/user/rwatson/netperf/sys/amd64/amd64/vm_machdep.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/amd64/amd64/vm_machdep.c 2005/01/19 20:22:17 @@ -142,7 +142,7 @@ pcb2->pcb_rsp = (register_t)td2->td_frame - sizeof(void *); pcb2->pcb_rbx = (register_t)td2; /* fork_trampoline argument */ pcb2->pcb_rip = (register_t)fork_trampoline; - pcb2->pcb_rflags = td2->td_frame->tf_rflags & ~PSL_I; /* ints disabled */ + pcb2->pcb_rflags = PSL_KERNEL; /* ints disabled */ /*- * pcb2->pcb_dr*: cloned above. * pcb2->pcb_savefpu: cloned above. @@ -151,6 +151,10 @@ * pcb2->pcb_[fg]sbase: cloned above */ + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_flags = pcb2->pcb_rflags | PSL_I; + /* * Now, cpu_switch() can schedule the new process. * pcb_rsp is loaded pointing to the cpu_switch() stack frame @@ -289,6 +293,10 @@ * pcb2->pcb_onfault: cloned above (always NULL here?). * pcb2->pcb_[fg]sbase: cloned above */ + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_flags = PSL_KERNEL | PSL_I; } /* --- //depot/user/rwatson/netperf/sys/amd64/include/proc.h 2004/04/07 03:50:45 +++ //depot/user/rwatson/percpu/sys/amd64/include/proc.h 2004/11/25 02:37:15 @@ -37,7 +37,8 @@ * Machine-dependent part of the proc structure for AMD64. */ struct mdthread { - register_t md_savecrit; + int md_spinlock_count; /* (k) */ + register_t md_saved_flags; /* (k) */ }; struct mdproc { --- //depot/user/rwatson/netperf/sys/arm/arm/machdep.c 2004/11/06 12:44:50 +++ //depot/user/rwatson/percpu/sys/arm/arm/machdep.c 2004/11/25 02:37:15 @@ -315,6 +315,32 @@ { } +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_cspr = disable_interrupts(I32_bit | F32_bit); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + restore_interrupts(td->td_md.md_saved_cspr); + } +} + /* * Clear registers on exec */ --- //depot/user/rwatson/netperf/sys/arm/arm/vm_machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/arm/arm/vm_machdep.c 2005/01/19 20:22:17 @@ -169,6 +169,10 @@ tf->tf_r0 = 0; tf->tf_r1 = 0; pcb2->un_32.pcb32_sp = (u_int)sf; + + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_cspr = 0; } void @@ -303,6 +307,10 @@ td->td_pcb->un_32.pcb32_sp = (u_int)sf; td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + td->td_kstack_pages * PAGE_SIZE + USPACE_UNDEF_STACK_TOP; + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_cspr = 0; } /* --- //depot/user/rwatson/netperf/sys/arm/include/proc.h 2004/05/23 16:56:02 +++ //depot/user/rwatson/percpu/sys/arm/include/proc.h 2004/11/25 02:37:15 @@ -46,7 +46,8 @@ }; struct mdthread { - register_t md_savecrit; + int md_spinlock_count; /* (k) */ + register_t md_saved_cspr; /* (k) */ }; struct mdproc { --- //depot/user/rwatson/netperf/sys/conf/files.alpha 2004/11/24 14:41:38 +++ //depot/user/rwatson/percpu/sys/conf/files.alpha 2004/11/25 02:37:15 @@ -43,7 +43,6 @@ alpha/alpha/clock.c standard alpha/alpha/clock_if.m standard alpha/alpha/cpuconf.c standard -alpha/alpha/critical.c standard alpha/alpha/db_disasm.c optional ddb alpha/alpha/db_interface.c optional ddb alpha/alpha/db_trace.c optional ddb --- //depot/user/rwatson/netperf/sys/conf/files.amd64 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/conf/files.amd64 2005/01/19 20:22:17 @@ -68,7 +68,6 @@ amd64/amd64/bios.c standard amd64/amd64/busdma_machdep.c standard amd64/amd64/cpu_switch.S standard -amd64/amd64/critical.c standard amd64/amd64/db_disasm.c optional ddb amd64/amd64/db_interface.c optional ddb amd64/amd64/db_trace.c optional ddb --- //depot/user/rwatson/netperf/sys/conf/files.arm 2004/11/24 14:41:38 +++ //depot/user/rwatson/percpu/sys/conf/files.arm 2004/11/25 02:37:15 @@ -12,7 +12,6 @@ arm/arm/cpufunc_asm_sa1.S standard arm/arm/cpufunc_asm_armv4.S standard arm/arm/cpufunc_asm_sa11x0.S standard -arm/arm/critical.c standard arm/arm/db_disasm.c optional ddb arm/arm/db_interface.c optional ddb arm/arm/db_trace.c optional ddb --- //depot/user/rwatson/netperf/sys/conf/files.i386 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/conf/files.i386 2005/01/19 20:22:17 @@ -220,7 +220,6 @@ i386/i386/bios.c standard i386/i386/bioscall.s standard i386/i386/busdma_machdep.c standard -i386/i386/critical.c standard i386/i386/db_disasm.c optional ddb i386/i386/db_interface.c optional ddb i386/i386/db_trace.c optional ddb --- //depot/user/rwatson/netperf/sys/conf/files.ia64 2004/11/24 14:41:38 +++ //depot/user/rwatson/percpu/sys/conf/files.ia64 2004/11/25 02:37:15 @@ -90,7 +90,6 @@ ia64/ia64/clock.c standard ia64/ia64/clock_if.m standard ia64/ia64/context.S standard -ia64/ia64/critical.c standard ia64/ia64/db_interface.c optional ddb ia64/ia64/db_trace.c optional ddb ia64/ia64/dump_machdep.c standard --- //depot/user/rwatson/netperf/sys/conf/files.pc98 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/conf/files.pc98 2005/01/19 20:22:17 @@ -142,7 +142,6 @@ i386/i386/bios.c standard i386/i386/bioscall.s standard i386/i386/busdma_machdep.c standard -i386/i386/critical.c standard i386/i386/db_disasm.c optional ddb i386/i386/db_interface.c optional ddb i386/i386/db_trace.c optional ddb --- //depot/user/rwatson/netperf/sys/conf/files.powerpc 2004/08/16 14:53:15 +++ //depot/user/rwatson/percpu/sys/conf/files.powerpc 2004/11/25 02:37:15 @@ -36,7 +36,6 @@ powerpc/powerpc/copyinout.c standard powerpc/powerpc/copystr.c standard powerpc/powerpc/cpu.c standard -powerpc/powerpc/critical.c standard powerpc/powerpc/elf_machdep.c standard powerpc/powerpc/fpu.c standard powerpc/powerpc/fuswintr.c standard --- //depot/user/rwatson/netperf/sys/conf/files.sparc64 2004/11/24 14:41:38 +++ //depot/user/rwatson/percpu/sys/conf/files.sparc64 2004/11/25 02:37:15 @@ -74,7 +74,6 @@ sparc64/sparc64/cheetah.c standard sparc64/sparc64/clock.c standard sparc64/sparc64/counter.c standard -sparc64/sparc64/critical.c standard sparc64/sparc64/db_disasm.c optional ddb sparc64/sparc64/db_interface.c optional ddb sparc64/sparc64/db_trace.c optional ddb --- //depot/user/rwatson/netperf/sys/i386/i386/machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/i386/i386/machdep.c 2005/01/19 20:22:17 @@ -2200,6 +2200,32 @@ pcpu->pc_acpi_id = 0xffffffff; } +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_flags = intr_disable(); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + intr_restore(td->td_md.md_saved_flags); + } +} + #if defined(I586_CPU) && !defined(NO_F00F_HACK) static void f00f_hack(void *unused); SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL) --- //depot/user/rwatson/netperf/sys/i386/i386/vm_machdep.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/i386/i386/vm_machdep.c 2005/01/19 20:22:17 @@ -256,6 +256,10 @@ } mtx_unlock_spin(&sched_lock); + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I; + /* * Now, cpu_switch() can schedule the new process. * pcb_esp is loaded pointing to the cpu_switch() stack frame @@ -364,7 +368,7 @@ /* * Initialize machine state (pcb and trap frame) for a new thread about to - * upcall. Pu t enough state in the new thread's PCB to get it to go back + * upcall. Put enough state in the new thread's PCB to get it to go back * userret(), where we can intercept it again to set the return (upcall) * Address and stack, along with those from upcals that are from other sources * such as those generated in thread_userret() itself. @@ -416,7 +420,7 @@ pcb2->pcb_esp = (int)td->td_frame - sizeof(void *); /* trampoline arg */ pcb2->pcb_ebx = (int)td; /* trampoline arg */ pcb2->pcb_eip = (int)fork_trampoline; - pcb2->pcb_psl &= ~(PSL_I); /* interrupts must be disabled */ + pcb2->pcb_psl = PSL_KERNEL; /* ints disabled */ pcb2->pcb_gs = rgs(); /* * If we didn't copy the pcb, we'd need to do the following registers: @@ -427,7 +431,11 @@ * pcb2->pcb_gs: cloned above. XXXKSE ??? * pcb2->pcb_ext: cleared below. */ - pcb2->pcb_ext = NULL; + pcb2->pcb_ext = NULL; + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_flags = PSL_KERNEL | PSL_I; } /* --- //depot/user/rwatson/netperf/sys/i386/include/proc.h 2004/06/28 19:42:50 +++ //depot/user/rwatson/percpu/sys/i386/include/proc.h 2004/11/25 02:37:15 @@ -47,7 +47,8 @@ * Machine-dependent part of the proc structure for i386. */ struct mdthread { - register_t md_savecrit; + int md_spinlock_count; /* (k) */ + register_t md_saved_flags; /* (k) */ }; struct mdproc { --- //depot/user/rwatson/netperf/sys/ia64/ia64/machdep.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/ia64/ia64/machdep.c 2005/01/19 20:22:17 @@ -381,6 +381,32 @@ } void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_intr = intr_disable(); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + intr_restore(td->td_md.md_saved_intr); + } +} + +void map_pal_code(void) { pt_entry_t pte; --- //depot/user/rwatson/netperf/sys/ia64/ia64/vm_machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/ia64/ia64/vm_machdep.c 2005/01/19 20:22:17 @@ -158,6 +158,10 @@ pcb->pcb_special.sp = (uintptr_t)tf - 16; pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline); cpu_set_fork_handler(td, (void (*)(void*))fork_return, td); + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_intr = 1; } void @@ -271,6 +275,10 @@ td2->td_pcb->pcb_special.sp = (uintptr_t)stackp - 16; td2->td_pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline); cpu_set_fork_handler(td2, (void (*)(void*))fork_return, td2); + + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_intr = 1; } /* --- //depot/user/rwatson/netperf/sys/ia64/include/proc.h 2004/02/28 22:29:37 +++ //depot/user/rwatson/percpu/sys/ia64/include/proc.h 2004/11/25 02:37:15 @@ -30,7 +30,8 @@ #define _MACHINE_PROC_H_ struct mdthread { - register_t md_savecrit; + int md_spinlock_count; /* (k) */ + register_t md_saved_intr; /* (k) */ }; struct mdproc { --- //depot/user/rwatson/netperf/sys/kern/kern_fork.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/kern/kern_fork.c 2005/01/19 20:22:17 @@ -72,7 +72,6 @@ #include #include -#include #ifndef _SYS_SYSPROTO_H_ struct fork_args { @@ -764,7 +763,6 @@ sched_lock.mtx_lock = (uintptr_t)td; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); - cpu_critical_fork_exit(); CTR4(KTR_PROC, "fork_exit: new thread %p (kse %p, pid %d, %s)", td, td->td_sched, p->p_pid, p->p_comm); --- //depot/user/rwatson/netperf/sys/kern/kern_mutex.c 2004/10/19 18:01:38 +++ //depot/user/rwatson/percpu/sys/kern/kern_mutex.c 2004/11/25 02:37:15 @@ -377,7 +377,7 @@ #if defined(SMP) || LOCK_DEBUG > 0 || 1 _get_spin_lock(m, curthread, opts, file, line); #else - critical_enter(); + spinlock_enter(); #endif LOCK_LOG_LOCK("LOCK", &m->mtx_object, opts, m->mtx_recurse, file, line); @@ -399,7 +399,7 @@ #if defined(SMP) || LOCK_DEBUG > 0 || 1 _rel_spin_lock(m); #else - critical_exit(); + spinlock_exit(); #endif } @@ -593,7 +593,7 @@ break; /* Give interrupts a chance while we spin. */ - critical_exit(); + spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 10000000) { cpu_spinwait(); @@ -612,7 +612,7 @@ } cpu_spinwait(); } - critical_enter(); + spinlock_enter(); } if (LOCK_LOG_TEST(&m->mtx_object, opts)) --- //depot/user/rwatson/netperf/sys/kern/kern_proc.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/kern/kern_proc.c 2005/01/19 20:22:17 @@ -63,7 +63,6 @@ #include #include #include -#include MALLOC_DEFINE(M_PGRP, "pgrp", "process group header"); MALLOC_DEFINE(M_SESSION, "session", "session header"); --- //depot/user/rwatson/netperf/sys/kern/kern_switch.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/kern/kern_switch.c 2005/01/19 20:22:17 @@ -105,7 +105,6 @@ #if defined(SMP) && (defined(__i386__) || defined(__amd64__)) #include #endif -#include #if defined(SMP) && defined(SCHED_4BSD) #include #endif @@ -568,8 +567,6 @@ struct thread *td; td = curthread; - if (td->td_critnest == 0) - cpu_critical_enter(td); td->td_critnest++; CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest); @@ -597,7 +594,6 @@ } #endif td->td_critnest = 0; - cpu_critical_exit(td); } else { td->td_critnest--; } --- //depot/user/rwatson/netperf/sys/pc98/i386/machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/pc98/i386/machdep.c 2005/01/19 20:22:17 @@ -2257,6 +2257,32 @@ } +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_flags = intr_disable(); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + intr_restore(td->td_md.md_saved_flags); + } +} + #if defined(I586_CPU) && !defined(NO_F00F_HACK) static void f00f_hack(void *unused); SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL) --- //depot/user/rwatson/netperf/sys/powerpc/include/cpufunc.h 2004/08/07 02:40:42 +++ //depot/user/rwatson/percpu/sys/powerpc/include/cpufunc.h 2004/11/25 02:37:15 @@ -149,8 +149,11 @@ static __inline void intr_restore(register_t msr) { + register_t value; - mtmsr(msr); + value = mfmsr() & ~(PSL_EE | PSL_RI); + value |= msr & (PSL_EE | PSL_RI); + mtmsr(value); } static __inline void --- //depot/user/rwatson/netperf/sys/powerpc/include/proc.h 2004/02/28 22:29:37 +++ //depot/user/rwatson/percpu/sys/powerpc/include/proc.h 2004/11/25 02:37:15 @@ -39,7 +39,8 @@ * Machine-dependent part of the proc structure */ struct mdthread { - register_t md_savecrit; + int md_spinlock_count; /* (k) */ + register_t md_saved_msr; /* (k) */ }; struct mdproc { --- //depot/user/rwatson/netperf/sys/powerpc/powerpc/machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/machdep.c 2005/01/19 20:22:17 @@ -875,6 +875,32 @@ } +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_msr = intr_disable(); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + intr_restore(td->td_md.md_saved_msr); + } +} + /* * kcopy(const void *src, void *dst, size_t len); * --- //depot/user/rwatson/netperf/sys/powerpc/powerpc/vm_machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/powerpc/powerpc/vm_machdep.c 2005/01/19 20:22:17 @@ -154,6 +154,10 @@ pcb->pcb_lr = (register_t)fork_trampoline; pcb->pcb_usr = kernel_pmap->pm_sr[USER_SR]; + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_msr = PSL_EE | PSL_RI; + /* * Now cpu_switch() can schedule the new process. */ @@ -329,6 +333,10 @@ pcb2->pcb_sp = (register_t)cf; pcb2->pcb_lr = (register_t)fork_trampoline; pcb2->pcb_usr = kernel_pmap->pm_sr[USER_SR]; + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_msr = PSL_EE | PSL_RI; } void --- //depot/user/rwatson/netperf/sys/sparc64/include/proc.h 2004/04/08 03:11:34 +++ //depot/user/rwatson/percpu/sys/sparc64/include/proc.h 2004/11/25 02:37:15 @@ -42,7 +42,8 @@ }; struct mdthread { - register_t md_savecrit; + int md_spinlock_count; /* (k) */ + register_t md_saved_pil; /* (k) */ }; struct mdproc { --- //depot/user/rwatson/netperf/sys/sparc64/sparc64/machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/sparc64/sparc64/machdep.c 2005/01/19 20:22:17 @@ -230,6 +230,33 @@ } } +void +spinlock_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_md.md_spinlock_count == 0) { + td->td_md.md_saved_pil = rdpr(pil); + wrpr(pil, 0, 14); + critical_enter(); + } + td->td_md.md_spinlock_count++; +} + +void +spinlock_exit(void) +{ + struct thread *td; + + td = curthread; + td->td_md.md_spinlock_count--; + if (td->td_md.md_spinlock_count == 0) { + critical_exit(); + wrpr(pil, td->td_md.md_saved_pil, 0); + } +} + unsigned tick_get_timecount(struct timecounter *tc) { --- //depot/user/rwatson/netperf/sys/sparc64/sparc64/vm_machdep.c 2004/11/27 21:43:45 +++ //depot/user/rwatson/percpu/sys/sparc64/sparc64/vm_machdep.c 2005/01/19 20:22:17 @@ -173,6 +173,10 @@ fr->fr_local[2] = (u_long)tf; pcb->pcb_pc = (u_long)fork_trampoline - 8; pcb->pcb_sp = (u_long)fr - SPOFF; + + /* Setup to release sched_lock in fork_exit(). */ + td->td_md.md_spinlock_count = 1; + td->td_md.md_saved_pil = 0; } void @@ -287,6 +291,10 @@ pcb2->pcb_sp = (u_long)fp - SPOFF; pcb2->pcb_pc = (u_long)fork_trampoline - 8; + /* Setup to release sched_lock in fork_exit(). */ + td2->td_md.md_spinlock_count = 1; + td2->td_md.md_saved_pil = 0; + /* * Now, cpu_switch() can schedule the new process. */ --- //depot/user/rwatson/netperf/sys/sys/lock.h 2004/02/28 22:29:37 +++ //depot/user/rwatson/percpu/sys/sys/lock.h 2004/11/25 02:37:15 @@ -196,6 +196,8 @@ extern struct lock_class lock_class_mtx_spin; extern struct lock_class lock_class_sx; +void spinlock_enter(void); +void spinlock_exit(void); void witness_init(struct lock_object *); void witness_destroy(struct lock_object *); int witness_defineorder(struct lock_object *, struct lock_object *); --- //depot/user/rwatson/netperf/sys/sys/mutex.h 2004/08/05 21:29:14 +++ //depot/user/rwatson/percpu/sys/sys/mutex.h 2004/11/25 02:37:15 @@ -164,7 +164,7 @@ #define _get_spin_lock(mp, tid, opts, file, line) do { \ struct thread *_tid = (tid); \ \ - critical_enter(); \ + spinlock_enter(); \ if (!_obtain_lock((mp), _tid)) { \ if ((mp)->mtx_lock == (uintptr_t)_tid) \ (mp)->mtx_recurse++; \ @@ -191,8 +191,8 @@ * Since spin locks are not _too_ common, inlining this code is not too big * a deal. * - * Since we always perform a critical_enter() when attempting to acquire a - * spin lock, we need to always perform a matching critical_exit() when + * Since we always perform a spinlock_enter() when attempting to acquire a + * spin lock, we need to always perform a matching spinlock_exit() when * releasing a spin lock. This includes the recursion cases. */ #ifndef _rel_spin_lock @@ -201,7 +201,7 @@ (mp)->mtx_recurse--; \ else \ _release_lock_quick((mp)); \ - critical_exit(); \ + spinlock_exit(); \ } while (0) #endif @@ -289,8 +289,8 @@ #define mtx_unlock_spin_flags(m, opts) \ _rel_spin_lock((m)) #else /* SMP */ -#define mtx_lock_spin_flags(m, opts) critical_enter() -#define mtx_unlock_spin_flags(m, opts) critical_exit() +#define mtx_lock_spin_flags(m, opts) spinlock_enter() +#define mtx_unlock_spin_flags(m, opts) spinlock_exit() #endif /* SMP */ #endif /* LOCK_DEBUG > 0 || MUTEX_NOINLINE */ --- //depot/user/rwatson/netperf/sys/vm/uma_core.c 2005/01/01 20:04:14 +++ //depot/user/rwatson/percpu/sys/vm/uma_core.c 2005/01/19 20:22:17 @@ -1,4 +1,5 @@ /* + * Copyright (c) 2004, Robert N. M. Watson * Copyright (c) 2004, 2005, * Bosko Milekic * Copyright (c) 2002, 2003, 2004, 2005, @@ -384,48 +385,19 @@ zone_timeout(uma_zone_t zone) { uma_keg_t keg; - uma_cache_t cache; u_int64_t alloc; - int cpu; keg = zone->uz_keg; alloc = 0; /* - * Aggregate per cpu cache statistics back to the zone. - * - * XXX This should be done in the sysctl handler. - * - * I may rewrite this to set a flag in the per cpu cache instead of - * locking. If the flag is not cleared on the next round I will have - * to lock and do it here instead so that the statistics don't get too - * far out of sync. - */ - if (!(keg->uk_flags & UMA_ZFLAG_INTERNAL)) { - for (cpu = 0; cpu <= mp_maxid; cpu++) { - if (CPU_ABSENT(cpu)) - continue; - CPU_LOCK(cpu); - cache = &zone->uz_cpu[cpu]; - /* Add them up, and reset */ - alloc += cache->uc_allocs; - cache->uc_allocs = 0; - CPU_UNLOCK(cpu); - } - } - - /* Now push these stats back into the zone.. */ - ZONE_LOCK(zone); - zone->uz_allocs += alloc; - - /* * Expand the zone hash table. * * This is done if the number of slabs is larger than the hash size. * What I'm trying to do here is completely reduce collisions. This * may be a little aggressive. Should I allow for two collisions max? */ - + ZONE_LOCK(zone); if (keg->uk_flags & UMA_ZONE_HASH && keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { struct uma_hash newhash; @@ -613,6 +585,10 @@ /* * Drains the per cpu caches for a zone. * + * NOTE: This may only be called while the zone is being turn down, and not + * during normal operation. This is necessary in order that we do not have + * to migrate CPUs to drain the per-CPU caches. + * * Arguments: * zone The zone to drain, must be unlocked. * @@ -626,12 +602,20 @@ int cpu; /* - * We have to lock each cpu cache before locking the zone + * XXX: It is safe to not lock the per-CPU caches, because we're + * tearing down the zone anyway. I.e., there will be no further use + * of the caches at this point. + * + * XXX: It would good to be able to assert that the zone is being + * torn down to prevent improper use of cache_drain(). + * + * XXX: We lock the zone before passing into bucket_cache_drain() as + * it is used elsewhere. Should the tear-down path be made special + * there in some form? */ for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) continue; - CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; bucket_drain(zone, cache->uc_allocbucket); bucket_drain(zone, cache->uc_freebucket); @@ -644,11 +628,6 @@ ZONE_LOCK(zone); bucket_cache_drain(zone); ZONE_UNLOCK(zone); - for (cpu = 0; cpu <= mp_maxid; cpu++) { - if (CPU_ABSENT(cpu)) - continue; - CPU_UNLOCK(cpu); - } } /* @@ -1786,6 +1765,9 @@ uma_cache_t cache; uma_bucket_t bucket; int cpu; +#ifdef INVARIANTS + int count; +#endif int badness; /* This is the fast path allocation */ @@ -1820,12 +1802,33 @@ } } + /* + * If possible, allocate from the per-CPU cache. There are two + * requirements for safe access to the per-CPU cache: (1) the thread + * accessing the cache must not be preempted or yield during access, + * and (2) the thread must not migrate CPUs without switching which + * cache it accesses. We rely on a critical section to prevent + * preemption and migration. We release the critical section in + * order to acquire the zone mutex if we are unable to free to the + * current cache; when we re-acquire the critical section, we must + * detect and handle migration if it has occurred. + */ +#ifdef INVARIANTS + count = 0; +#endif zalloc_restart: + critical_enter(); cpu = PCPU_GET(cpuid); - CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; zalloc_start: +#ifdef INVARIANTS + count++; + KASSERT(count < 10, ("uma_zalloc_arg: count == 10")); +#endif +#if 0 + critical_assert(); +#endif bucket = cache->uc_allocbucket; if (bucket) { @@ -1838,12 +1841,12 @@ KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); cache->uc_allocs++; + critical_exit(); #ifdef INVARIANTS ZONE_LOCK(zone); uma_dbg_alloc(zone, NULL, item); ZONE_UNLOCK(zone); #endif - CPU_UNLOCK(cpu); if (zone->uz_ctor != NULL) { if (zone->uz_ctor(item, zone->uz_keg->uk_size, udata, flags) != 0) { @@ -1873,7 +1876,33 @@ } } } + /* + * Attempt to retrieve the item from the per-CPU cache has failed, so + * we must go back to the zone. This requires the zone lock, so we + * must drop the critical section, then re-acquire it when we go back + * to the cache. Since the critical section is released, we may be + * preempted or migrate. As such, make sure not to maintain any + * thread-local state specific to the cache from prior to releasing + * the critical section. + */ + critical_exit(); ZONE_LOCK(zone); + critical_enter(); + cpu = PCPU_GET(cpuid); + cache = &zone->uz_cpu[cpu]; + bucket = cache->uc_allocbucket; + if (bucket != NULL) { + if (bucket != NULL && bucket->ub_cnt > 0) { + ZONE_UNLOCK(zone); + goto zalloc_start; + } + bucket = cache->uc_freebucket; + if (bucket != NULL && bucket->ub_cnt > 0) { + ZONE_UNLOCK(zone); + goto zalloc_start; + } + } + /* Since we have locked the zone we may as well send back our stats */ zone->uz_allocs += cache->uc_allocs; cache->uc_allocs = 0; @@ -1897,8 +1926,8 @@ ZONE_UNLOCK(zone); goto zalloc_start; } - /* We are no longer associated with this cpu!!! */ - CPU_UNLOCK(cpu); + /* We are no longer associated with this CPU. */ + critical_exit(); /* Bump up our uz_count so we get here less */ if (zone->uz_count < BUCKET_MAX) @@ -2211,6 +2240,9 @@ uma_bucket_t bucket; int bflags; int cpu; +#ifdef INVARIANTS + int count; +#endif enum zfreeskip skip; /* This is the fast path free */ @@ -2236,12 +2268,33 @@ skip = SKIP_DTOR; } +#ifdef INVARIANTS + count = 0; +#endif + /* + * If possible, free to the per-CPU cache. There are two + * requirements for safe access to the per-CPU cache: (1) the thread + * accessing the cache must not be preempted or yield during access, + * and (2) the thread must not migrate CPUs without switching which + * cache it accesses. We rely on a critical section to prevent + * preemption and migration. We release the critical section in + * order to acquire the zone mutex if we are unable to free to the + * current cache; when we re-acquire the critical section, we must + * detect and handle migration if it has occurred. + */ zfree_restart: + critical_enter(); cpu = PCPU_GET(cpuid); - CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; zfree_start: +#ifdef INVARIANTS + count++; + KASSERT(count < 10, ("uma_zfree_arg: count == 10")); +#endif +#if 0 + critical_assert(); +#endif bucket = cache->uc_freebucket; if (bucket) { @@ -2255,6 +2308,7 @@ ("uma_zfree: Freeing to non free bucket index.")); bucket->ub_bucket[bucket->ub_cnt] = item; bucket->ub_cnt++; + critical_exit(); #ifdef INVARIANTS ZONE_LOCK(zone); if (keg->uk_flags & UMA_ZONE_MALLOC) @@ -2263,7 +2317,6 @@ uma_dbg_free(zone, NULL, item); ZONE_UNLOCK(zone); #endif - CPU_UNLOCK(cpu); return; } else if (cache->uc_allocbucket) { #ifdef UMA_DEBUG_ALLOC @@ -2287,9 +2340,32 @@ * * 1) The buckets are NULL * 2) The alloc and free buckets are both somewhat full. + * + * We must go back the zone, hich requires acquiring the zone lock, + * which in turn means we must release and re-acquire the critical + * section. Since the critical section is released, we may be + * preempted or migrate. As such, make sure not to maintain any + * thread-local state specific to the cache from prior to releasing + * the critical section. */ - + critical_exit(); ZONE_LOCK(zone); + critical_enter(); + cpu = PCPU_GET(cpuid); + cache = &zone->uz_cpu[cpu]; + if (cache->uc_freebucket != NULL) { + if (cache->uc_freebucket->ub_cnt < + cache->uc_freebucket->ub_entries) { + ZONE_UNLOCK(zone); + goto zfree_start; + } + if (cache->uc_allocbucket != NULL && + (cache->uc_allocbucket->ub_cnt < + cache->uc_freebucket->ub_cnt)) { + ZONE_UNLOCK(zone); + goto zfree_start; + } + } bucket = cache->uc_freebucket; cache->uc_freebucket = NULL; @@ -2311,8 +2387,8 @@ cache->uc_freebucket = bucket; goto zfree_start; } - /* We're done with this CPU now */ - CPU_UNLOCK(cpu); + /* We are nolonger associated with this CPU. */ + critical_exit(); /* And the zone.. */ ZONE_UNLOCK(zone); @@ -2726,6 +2802,7 @@ int cachefree; uma_bucket_t bucket; uma_cache_t cache; + u_int64_t alloc; cnt = 0; mtx_lock(&uma_mtx); @@ -2749,6 +2826,13 @@ LIST_FOREACH(z, &zk->uk_zones, uz_link) { if (cnt == 0) /* list may have changed size */ break; + /* + * XXXRW: This is the last remaining use of a per-CPU mutex + * to protect access to a per-CPU zone cache. We should be + * migrating CPUs here to pick up data from each cache + * instead. Because this locking no longer protects other + * use of the cache, we can now get inconsistent results. + */ if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) { for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) @@ -2758,6 +2842,7 @@ } ZONE_LOCK(z); cachefree = 0; + alloc = 0; if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) { for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) @@ -2767,9 +2852,15 @@ cachefree += cache->uc_allocbucket->ub_cnt; if (cache->uc_freebucket != NULL) cachefree += cache->uc_freebucket->ub_cnt; + alloc += cache->uc_allocs; + cache->uc_allocs = 0; CPU_UNLOCK(cpu); } } + + /* Now push alloc stats back into zone. */ + z->uz_allocs += alloc; + LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) { cachefree += bucket->ub_cnt; }