Divorce critical sections from spinlocks. Critical sections as denoted by

critical_enter() and critical_exit() are now solely a mechanism for
deferring kernel preemptions.  They no longer have any affect on
interrupts.  This means that standalone critical sections are now very
cheap as they are simply unlocked integer increments and decrements for the
common case.

Spin mutexes now use a separate KPI implemented in MD code: spinlock_enter()
and spinlock_exit().  This KPI is responsible for providing whatever MD
guarantees are needed to ensure that a thread holding a spin lock won't
be preempted by any other code that will try to lock the same lock.  For
now all archs continue to block interrupts in a "spinlock section" as they
did formerly in all critical sections.  Note that I've also taken this
opportunity to push a few things into MD code rather than MI.  For example,
critical_fork_exit() no longer exists.  Instead, MD code ensures that new
threads have the correct state when they are created.  Also, we no longer
try to fixup the idlethreads for APs in MI code.  Instead, each arch sets
the initial curthread and adjusts the state of the idle thread it borrows
in order to perform the initial context switch.

This change is largely a big NOP, but the cleaner separation it provides
will allow for more efficient alternative locking schemes in other parts
of the kernel (bare critical sections rather than per-CPU spin mutexes
for per-CPU data for example).

Reviewed by:	grehan, cognet, arch@, others
Tested on:	i386, alpha, sparc64, powerpc, arm, possibly more

61 files changed, 408 insertions, 996 deletions

diff --git a/sys/alpha/alpha/critical.c b/sys/alpha/alpha/critical.c
deleted file mode 100644
index 52aac72f5c6a..000000000000
--- a/sys/alpha/alpha/critical.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/pcpu.h>
-#include <sys/eventhandler.h>	/* XX */
-#include <sys/ktr.h>		/* XX */
-#include <sys/signalvar.h>
-#include <sys/sysproto.h>	/* XX */
-#include <sys/kernel.h>
-#include <sys/proc.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/sysctl.h>
-#include <sys/ucontext.h>
-#include <machine/critical.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- *
- */
-void
-cpu_critical_fork_exit(void)
-{
-	struct thread *td;
-
-	td = curthread;
-	td->td_md.md_savecrit = ALPHA_PSL_IPL_0;
-}
diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c
index f833e400cf26..e09b5f692b93 100644
--- a/sys/alpha/alpha/machdep.c
+++ b/sys/alpha/alpha/machdep.c
@@ -2397,3 +2397,27 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz)
 	pcpu->pc_idlepcb.apcb_ptbr = thread0.td_pcb->pcb_hw.apcb_ptbr;
 	pcpu->pc_current_asngen = 1;
 }
+
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_ipl = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_ipl);
+}
diff --git a/sys/alpha/alpha/mp_machdep.c b/sys/alpha/alpha/mp_machdep.c
index 2380c90379e4..c1b9706d8118 100644
--- a/sys/alpha/alpha/mp_machdep.c
+++ b/sys/alpha/alpha/mp_machdep.c
@@ -144,6 +144,10 @@ smp_init_secondary(void)
 	/* Clear userland thread pointer. */
 	alpha_pal_wrunique(0);
 
+	/* Initialize curthread. */
+	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
+	PCPU_SET(curthread, PCPU_GET(idlethread));
+
 	/*
 	 * Point interrupt/exception vectors to our own.
 	 */
@@ -205,11 +209,24 @@ smp_init_secondary(void)
 	while (smp_started == 0)
 		; /* nothing */
 
+	/* ok, now grab sched_lock and enter the scheduler */
+	mtx_lock_spin(&sched_lock);
+
+	/*
+	 * Correct spinlock nesting.  The idle thread context that we are
+	 * borrowing was created so that it would start out with a single
+	 * spin lock (sched_lock) held in fork_trampoline().  Since we've
+	 * explicitly acquired locks in this function, the nesting count
+	 * is now 2 rather than 1.  Since we are nested, calling
+	 * spinlock_exit() will simply adjust the counts without allowing
+	 * spin lock using code to interrupt us.
+	 */
+	spinlock_exit();
+	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
+
 	binuptime(PCPU_PTR(switchtime));
 	PCPU_SET(switchticks, ticks);
 
-	/* ok, now grab sched_lock and enter the scheduler */
-	mtx_lock_spin(&sched_lock);
 	cpu_throw(NULL, choosethread());	/* doesn't return */
 
 	panic("scheduler returned us to %s", __func__);
diff --git a/sys/alpha/alpha/vm_machdep.c b/sys/alpha/alpha/vm_machdep.c
index 17972b9b28f6..be96f9b0f8ed 100644
--- a/sys/alpha/alpha/vm_machdep.c
+++ b/sys/alpha/alpha/vm_machdep.c
@@ -202,6 +202,10 @@ cpu_fork(td1, p2, td2, flags)
 	 */
 	td2->td_md.md_kernnest = 1;
 #endif
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_ipl = ALPHA_PSL_IPL_0;
 }
 
 /*
@@ -319,6 +323,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	 */
 	td->td_md.md_kernnest = 1;
 #endif
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_ipl = ALPHA_PSL_IPL_0;
 }
 
 void
diff --git a/sys/alpha/include/critical.h b/sys/alpha/include/critical.h
deleted file mode 100644
index 4c9f729dfc23..000000000000
--- a/sys/alpha/include/critical.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef _MACHINE_CRITICAL_H_
-#define	_MACHINE_CRITICAL_H_
-
-__BEGIN_DECLS
-
-/*
- * Prototypes - see <arch>/<arch>/critical.c
- */
-void cpu_critical_fork_exit(void);
-
-#ifdef	__CC_SUPPORTS___INLINE
-
-/*
- *	cpu_critical_enter:
- *
- *	This routine is called from critical_enter() on the 0->1 transition
- *	of td_critnest, prior to it being incremented to 1.
- */
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-
-	td->td_md.md_savecrit = intr_disable();
-}
-
-/*
- *	cpu_critical_exit:
- *
- *	This routine is called from critical_exit() on a 1->0 transition
- *	of td_critnest, after it has been decremented to 0.  We are
- *	exiting the last critical section.
- */
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-
-	intr_restore(td->td_md.md_savecrit);
-}
-
-#else /* !__CC_SUPPORTS___INLINE */
-
-void cpu_critical_enter(struct thread *td);
-void cpu_critical_exit(struct thread *td);
-
-#endif	/* __CC_SUPPORTS___INLINE */
-
-__END_DECLS
-
-#endif /* !_MACHINE_CRITICAL_H_ */
-
diff --git a/sys/alpha/include/proc.h b/sys/alpha/include/proc.h
index be3948aa9352..174e64bde9aa 100644
--- a/sys/alpha/include/proc.h
+++ b/sys/alpha/include/proc.h
@@ -52,7 +52,8 @@ struct mdthread {
 	u_int64_t	md_hae;		/* user HAE register value */
 	void            *osf_sigtramp;  /* user-level signal trampoline */
 	u_int		md_kernnest;	/* nesting level in the kernel */
-	register_t	md_savecrit;	/* save PSL for critical section */
+	register_t	md_saved_ipl;	/* save IPL for critical section */
+	u_int		md_spinlock_count;
 };
 
 #define	MDP_UAC_NOPRINT	0x0010		/* Don't print unaligned traps */
diff --git a/sys/amd64/amd64/critical.c b/sys/amd64/amd64/critical.c
deleted file mode 100644
index 925a91ca2865..000000000000
--- a/sys/amd64/amd64/critical.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/proc.h>
-
-#include <machine/critical.h>
-#include <machine/psl.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- *
- *	Enable interrupts in the saved copy of eflags.
- */
-void
-cpu_critical_fork_exit(void)
-{
-
-	curthread->td_md.md_savecrit = read_rflags() | PSL_I;
-}
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index da9e5fd834b5..9ee122951f92 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -1302,6 +1302,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 	pcpu->pc_acpi_id = 0xffffffff;
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_flags = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_flags);
+}
+
 /*
  * Construct a PCB from a trapframe. This is called from kdb_trap() where
  * we want to start a backtrace from the function that caused us to enter
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index fa6723bb4d8d..aab6e8b55eae 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -452,6 +452,10 @@ init_secondary(void)
 		panic("cpuid mismatch! boom!!");
 	}
 
+	/* Initialize curthread. */
+	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
+	PCPU_SET(curthread, PCPU_GET(idlethread));
+
 	mtx_lock_spin(&ap_boot_mtx);
 
 	/* Init local apic for irq's */
@@ -490,6 +494,18 @@ init_secondary(void)
 	/* ok, now grab sched_lock and enter the scheduler */
 	mtx_lock_spin(&sched_lock);
 
+	/*
+	 * Correct spinlock nesting.  The idle thread context that we are
+	 * borrowing was created so that it would start out with a single
+	 * spin lock (sched_lock) held in fork_trampoline().  Since we've
+	 * explicitly acquired locks in this function, the nesting count
+	 * is now 2 rather than 1.  Since we are nested, calling
+	 * spinlock_exit() will simply adjust the counts without allowing
+	 * spin lock using code to interrupt us.
+	 */
+	spinlock_exit();
+	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
+
 	binuptime(PCPU_PTR(switchtime));
 	PCPU_SET(switchticks, ticks);
 
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index 9b9981234aef..dd1e4bf35266 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -163,6 +163,10 @@ cpu_fork(td1, p2, td2, flags)
 	 * pcb2->pcb_[fg]sbase:	cloned above
 	 */
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_rsp is loaded pointing to the cpu_switch() stack frame
@@ -294,6 +298,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	 * pcb2->pcb_onfault:	cloned above (always NULL here?).
 	 * pcb2->pcb_[fg]sbase: cloned above
 	 */
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 }
 
 /*
diff --git a/sys/amd64/include/critical.h b/sys/amd64/include/critical.h
deleted file mode 100644
index ac85f2f452c2..000000000000
--- a/sys/amd64/include/critical.h
+++ /dev/null
@@ -1,87 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef _MACHINE_CRITICAL_H_
-#define	_MACHINE_CRITICAL_H_
-
-__BEGIN_DECLS
-
-/*
- * Prototypes - see <arch>/<arch>/critical.c
- */
-void cpu_critical_fork_exit(void);
-
-#ifdef __CC_SUPPORTS___INLINE
-
-/*
- *	cpu_critical_enter:
- *
- *	This routine is called from critical_enter() on the 0->1 transition
- *	of td_critnest, prior to it being incremented to 1.
- */
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-
-	td->td_md.md_savecrit = intr_disable();
-}
-
-/*
- *	cpu_critical_exit:
- *
- *	This routine is called from critical_exit() on a 1->0 transition
- *	of td_critnest, after it has been decremented to 0.  We are
- *	exiting the last critical section.
- */
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-	intr_restore(td->td_md.md_savecrit);
-}
-
-#else /* !__CC_SUPPORTS___INLINE */
-
-void cpu_critical_enter(struct thread *td);
-void cpu_critical_exit(struct thread *td);
-
-#endif        /* __CC_SUPPORTS___INLINE */
-
-__END_DECLS
-
-#endif /* !_MACHINE_CRITICAL_H_ */
-
diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h
index b019455e5cd1..ba6354b68dce 100644
--- a/sys/amd64/include/proc.h
+++ b/sys/amd64/include/proc.h
@@ -37,7 +37,8 @@
  * Machine-dependent part of the proc structure for AMD64.
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_flags;	/* (k) */
 };
 
 struct mdproc {
diff --git a/sys/arm/arm/critical.c b/sys/arm/arm/critical.c
deleted file mode 100644
index 1c6d0fa48fc1..000000000000
--- a/sys/arm/arm/critical.c
+++ /dev/null
@@ -1,52 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/signalvar.h>
-#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/pcpu.h>
-#include <sys/proc.h>
-#include <sys/sysctl.h>
-#include <sys/ucontext.h>
-#include <machine/clock.h>
-#include <machine/critical.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- */
-void
-cpu_critical_fork_exit(void)
-{
-
-	curthread->td_md.md_savecrit = __set_cpsr_c(0, 0) &~ I32_bit;
-}
-
diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c
index 4ff4d295dbf2..6d8313dbe61a 100644
--- a/sys/arm/arm/machdep.c
+++ b/sys/arm/arm/machdep.c
@@ -390,6 +390,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 {
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_cspr = disable_interrupts(I32_bit | F32_bit);
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		restore_interrupts(td->td_md.md_saved_cspr);
+}
+
 /*
  * Clear registers on exec
  */
diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c
index 586e220afe3c..a90cf62a6d5f 100644
--- a/sys/arm/arm/vm_machdep.c
+++ b/sys/arm/arm/vm_machdep.c
@@ -129,6 +129,10 @@ cpu_fork(register struct thread *td1, register struct proc *p2,
 	tf->tf_r0 = 0;
 	tf->tf_r1 = 0;
 	pcb2->un_32.pcb32_sp = (u_int)sf;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_cspr = 0;
 }
 				
 void
@@ -263,6 +267,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	td->td_pcb->un_32.pcb32_sp = (u_int)sf;
 	td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + td->td_kstack_pages
 	    * PAGE_SIZE + USPACE_UNDEF_STACK_TOP;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_cspr = 0;
 }
 
 /*
diff --git a/sys/arm/include/critical.h b/sys/arm/include/critical.h
deleted file mode 100644
index f9f057814458..000000000000
--- a/sys/arm/include/critical.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef MACHINE_CRITICAL_H
-#define MACHINE_CRITICAL_H
-void cpu_critical_fork_exit(void);
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-	td->td_md.md_savecrit = disable_interrupts(I32_bit | F32_bit);
-}
-
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-	restore_interrupts(td->td_md.md_savecrit);
-}
-
-#endif
diff --git a/sys/arm/include/proc.h b/sys/arm/include/proc.h
index eb548ccedd64..597c75ceaed6 100644
--- a/sys/arm/include/proc.h
+++ b/sys/arm/include/proc.h
@@ -46,7 +46,8 @@ struct md_utrap {
 };
 
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_cspr;	/* (k) */
 	int md_ptrace_instr;
 	int md_ptrace_addr;
 	void *md_tp;
diff --git a/sys/conf/files.alpha b/sys/conf/files.alpha
index b256fd4ef2b5..8fee17a0f217 100644
--- a/sys/conf/files.alpha
+++ b/sys/conf/files.alpha
@@ -43,7 +43,6 @@ alpha/alpha/busspace.c		standard
 alpha/alpha/clock.c		standard
 alpha/alpha/clock_if.m		standard
 alpha/alpha/cpuconf.c		standard
-alpha/alpha/critical.c		standard
 alpha/alpha/db_disasm.c		optional	ddb
 alpha/alpha/db_interface.c	optional	ddb
 alpha/alpha/db_trace.c		optional	ddb
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index ab65a6e3ab92..30da61b59097 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -78,7 +78,6 @@ amd64/amd64/autoconf.c		standard
 amd64/amd64/bios.c		standard
 amd64/amd64/busdma_machdep.c	standard
 amd64/amd64/cpu_switch.S	standard
-amd64/amd64/critical.c		standard
 amd64/amd64/db_disasm.c		optional	ddb
 amd64/amd64/db_interface.c	optional	ddb
 amd64/amd64/db_trace.c		optional	ddb
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 7bc73b570c95..2a7dfeb0316c 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -12,7 +12,6 @@ arm/arm/cpufunc_asm.S		standard
 arm/arm/cpufunc_asm_sa1.S 	standard
 arm/arm/cpufunc_asm_armv4.S 	standard
 arm/arm/cpufunc_asm_sa11x0.S 	standard
-arm/arm/critical.c		standard
 arm/arm/db_disasm.c		optional	ddb
 arm/arm/db_interface.c		optional	ddb
 arm/arm/db_trace.c		optional	ddb
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index aad236488b4f..fc5ae3abdc06 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -241,7 +241,6 @@ i386/i386/autoconf.c		standard
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard
 i386/i386/busdma_machdep.c	standard
-i386/i386/critical.c		standard
 i386/i386/db_disasm.c		optional ddb
 i386/i386/db_interface.c	optional ddb
 i386/i386/db_trace.c		optional ddb
diff --git a/sys/conf/files.ia64 b/sys/conf/files.ia64
index 7d2cb4302e29..6a050f843900 100644
--- a/sys/conf/files.ia64
+++ b/sys/conf/files.ia64
@@ -90,7 +90,6 @@ ia64/ia64/busdma_machdep.c	standard
 ia64/ia64/clock.c		standard
 ia64/ia64/clock_if.m		standard
 ia64/ia64/context.S		standard
-ia64/ia64/critical.c		standard
 ia64/ia64/db_interface.c	optional	ddb
 ia64/ia64/db_trace.c		optional	ddb
 ia64/ia64/dump_machdep.c	standard
diff --git a/sys/conf/files.pc98 b/sys/conf/files.pc98
index f650473ae435..fe68f9b8d4de 100644
--- a/sys/conf/files.pc98
+++ b/sys/conf/files.pc98
@@ -162,7 +162,6 @@ i386/i386/autoconf.c		standard
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard
 i386/i386/busdma_machdep.c	standard
-i386/i386/critical.c		standard
 i386/i386/db_disasm.c		optional ddb
 i386/i386/db_interface.c	optional ddb
 i386/i386/db_trace.c		optional ddb
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index ed52bcb20ab2..d3004c3dbdb7 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -36,7 +36,6 @@ powerpc/powerpc/clock.c		standard
 powerpc/powerpc/copyinout.c	standard
 powerpc/powerpc/copystr.c	standard
 powerpc/powerpc/cpu.c		standard
-powerpc/powerpc/critical.c	standard
 powerpc/powerpc/elf_machdep.c	standard
 powerpc/powerpc/fpu.c		standard
 powerpc/powerpc/fuswintr.c	standard
diff --git a/sys/conf/files.sparc64 b/sys/conf/files.sparc64
index a7d4691ae1c4..86157aef7f98 100644
--- a/sys/conf/files.sparc64
+++ b/sys/conf/files.sparc64
@@ -73,7 +73,6 @@ sparc64/sparc64/cache.c		standard
 sparc64/sparc64/cheetah.c	standard
 sparc64/sparc64/clock.c		standard
 sparc64/sparc64/counter.c	standard
-sparc64/sparc64/critical.c	standard
 sparc64/sparc64/db_disasm.c	optional	ddb
 sparc64/sparc64/db_interface.c	optional	ddb
 sparc64/sparc64/db_trace.c	optional	ddb
diff --git a/sys/dev/sio/sio_pci.c b/sys/dev/sio/sio_pci.c
index 05d42268f655..a59e2212155e 100644
--- a/sys/dev/sio/sio_pci.c
+++ b/sys/dev/sio/sio_pci.c
@@ -71,6 +71,7 @@ struct pci_ids {
 static struct pci_ids pci_ids[] = {
 	{ 0x100812b9, "3COM PCI FaxModem", 0x10 },
 	{ 0x2000131f, "CyberSerial (1-port) 16550", 0x10 },
+	{ 0x65851282, "Davicom 56PDV PCI Modem", 0x10 },
 	{ 0x01101407, "Koutech IOFLEX-2S PCI Dual Port Serial", 0x10 },
 	{ 0x01111407, "Koutech IOFLEX-2S PCI Dual Port Serial", 0x10 },
 	{ 0x048011c1, "Lucent kermit based PCI Modem", 0x14 },
diff --git a/sys/i386/i386/critical.c b/sys/i386/i386/critical.c
deleted file mode 100644
index 77965d413987..000000000000
--- a/sys/i386/i386/critical.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/proc.h>
-
-#include <machine/critical.h>
-#include <machine/psl.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- *
- *	Enable interrupts in the saved copy of eflags.
- */
-void
-cpu_critical_fork_exit(void)
-{
-
-	curthread->td_md.md_savecrit = read_eflags() | PSL_I;
-}
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index b759a7fbaa2e..bbd3fc1931c0 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -2219,6 +2219,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 	pcpu->pc_acpi_id = 0xffffffff;
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_flags = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_flags);
+}
+
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 static void f00f_hack(void *unused);
 SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL)
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 7cd184a22b02..bb316e87e5a0 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -432,6 +432,7 @@ cpu_mp_announce(void)
 void
 init_secondary(void)
 {
+	vm_offset_t addr;
 	int	gsel_tss;
 	int	x, myid;
 	u_int	cr0;
@@ -489,7 +490,8 @@ init_secondary(void)
 
 	/* BSP may have changed PTD while we were waiting */
 	invltlb();
-	pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
+	for (addr = 0; addr < NKPT * NBPDR - 1; addr += PAGE_SIZE)
+		invlpg(addr);
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 	lidt(&r_idt);
@@ -513,6 +515,10 @@ init_secondary(void)
 		panic("cpuid mismatch! boom!!");
 	}
 
+	/* Initialize curthread. */
+	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
+	PCPU_SET(curthread, PCPU_GET(idlethread));
+
 	mtx_lock_spin(&ap_boot_mtx);
 
 	/* Init local apic for irq's */
@@ -551,6 +557,18 @@ init_secondary(void)
 	/* ok, now grab sched_lock and enter the scheduler */
 	mtx_lock_spin(&sched_lock);
 
+	/*
+	 * Correct spinlock nesting.  The idle thread context that we are
+	 * borrowing was created so that it would start out with a single
+	 * spin lock (sched_lock) held in fork_trampoline().  Since we've
+	 * explicitly acquired locks in this function, the nesting count
+	 * is now 2 rather than 1.  Since we are nested, calling
+	 * spinlock_exit() will simply adjust the counts without allowing
+	 * spin lock using code to interrupt us.
+	 */
+	spinlock_exit();
+	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
+
 	binuptime(PCPU_PTR(switchtime));
 	PCPU_SET(switchticks, ticks);
 
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index 432344246fd9..033b8d86f8ce 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -257,6 +257,10 @@ cpu_fork(td1, p2, td2, flags)
 	}
 	mtx_unlock_spin(&sched_lock);
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_esp is loaded pointing to the cpu_switch() stack frame
@@ -423,6 +427,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	 * pcb2->pcb_ext:	cleared below.
 	 */
 	pcb2->pcb_ext = NULL;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 }
 
 /*
diff --git a/sys/i386/include/critical.h b/sys/i386/include/critical.h
deleted file mode 100644
index ac85f2f452c2..000000000000
--- a/sys/i386/include/critical.h
+++ /dev/null
@@ -1,87 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef _MACHINE_CRITICAL_H_
-#define	_MACHINE_CRITICAL_H_
-
-__BEGIN_DECLS
-
-/*
- * Prototypes - see <arch>/<arch>/critical.c
- */
-void cpu_critical_fork_exit(void);
-
-#ifdef __CC_SUPPORTS___INLINE
-
-/*
- *	cpu_critical_enter:
- *
- *	This routine is called from critical_enter() on the 0->1 transition
- *	of td_critnest, prior to it being incremented to 1.
- */
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-
-	td->td_md.md_savecrit = intr_disable();
-}
-
-/*
- *	cpu_critical_exit:
- *
- *	This routine is called from critical_exit() on a 1->0 transition
- *	of td_critnest, after it has been decremented to 0.  We are
- *	exiting the last critical section.
- */
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-	intr_restore(td->td_md.md_savecrit);
-}
-
-#else /* !__CC_SUPPORTS___INLINE */
-
-void cpu_critical_enter(struct thread *td);
-void cpu_critical_exit(struct thread *td);
-
-#endif        /* __CC_SUPPORTS___INLINE */
-
-__END_DECLS
-
-#endif /* !_MACHINE_CRITICAL_H_ */
-
diff --git a/sys/i386/include/proc.h b/sys/i386/include/proc.h
index 2ac31fb82a40..ec2e6b4beefd 100644
--- a/sys/i386/include/proc.h
+++ b/sys/i386/include/proc.h
@@ -47,7 +47,8 @@ struct proc_ldt {
  * Machine-dependent part of the proc structure for i386.
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_flags;	/* (k) */
 };
 
 struct mdproc {
diff --git a/sys/ia64/ia64/critical.c b/sys/ia64/ia64/critical.c
deleted file mode 100644
index af64b6e2effa..000000000000
--- a/sys/ia64/ia64/critical.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/pcpu.h>
-#include <sys/eventhandler.h>	/* XX */
-#include <sys/ktr.h>		/* XX */
-#include <sys/signalvar.h>
-#include <sys/sysproto.h>	/* XX */
-#include <sys/kernel.h>
-#include <sys/proc.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/sysctl.h>
-#include <sys/ucontext.h>
-#include <machine/critical.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- */
-void
-cpu_critical_fork_exit(void)
-{
-	struct thread *td;
-
-	td = curthread;
-	td->td_md.md_savecrit = (ia64_get_psr() | IA64_PSR_I);
-}
diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index e9866f53003f..82a5423e355b 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -401,6 +401,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 }
 
 void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_intr = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_intr);
+}
+
+void
 map_pal_code(void)
 {
 	pt_entry_t pte;
diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c
index b7efc76c238e..0c4bdf7a8c8c 100644
--- a/sys/ia64/ia64/mp_machdep.c
+++ b/sys/ia64/ia64/mp_machdep.c
@@ -111,17 +111,33 @@ ia64_ap_startup(void)
 	ia64_mca_save_state(SAL_INFO_MCA);
 	ia64_mca_save_state(SAL_INFO_CMC);
 
+	/* Initialize curthread. */
+	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
+	PCPU_SET(curthread, PCPU_GET(idlethread));
+
 	ap_awake++;
 	while (!smp_started)
 		/* spin */;
 
 	CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid));
 
+	mtx_lock_spin(&sched_lock);
+
+	/*
+	 * Correct spinlock nesting.  The idle thread context that we are
+	 * borrowing was created so that it would start out with a single
+	 * spin lock (sched_lock) held in fork_trampoline().  Since we've
+	 * explicitly acquired locks in this function, the nesting count
+	 * is now 2 rather than 1.  Since we are nested, calling
+	 * spinlock_exit() will simply adjust the counts without allowing
+	 * spin lock using code to interrupt us.
+	 */
+	spinlock_exit();
+	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
+
 	binuptime(PCPU_PTR(switchtime));
 	PCPU_SET(switchticks, ticks);
 
-	mtx_lock_spin(&sched_lock);
-
 	ia64_set_tpr(0);
 
 	/* kick off the clock on this AP */
diff --git a/sys/ia64/ia64/vm_machdep.c b/sys/ia64/ia64/vm_machdep.c
index f8fdb9498930..b09814e1de13 100644
--- a/sys/ia64/ia64/vm_machdep.c
+++ b/sys/ia64/ia64/vm_machdep.c
@@ -158,6 +158,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	pcb->pcb_special.sp = (uintptr_t)tf - 16;
 	pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
 	cpu_set_fork_handler(td, (void (*)(void*))fork_return, td);
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_intr = 1;
 }
 
 void
@@ -271,6 +275,10 @@ cpu_fork(struct thread *td1, struct proc *p2 __unused, struct thread *td2,
 	td2->td_pcb->pcb_special.sp = (uintptr_t)stackp - 16;
 	td2->td_pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
 	cpu_set_fork_handler(td2, (void (*)(void*))fork_return, td2);
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_intr = 1;
 }
 
 /*
diff --git a/sys/ia64/include/critical.h b/sys/ia64/include/critical.h
deleted file mode 100644
index 26f2edd4e796..000000000000
--- a/sys/ia64/include/critical.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef _MACHINE_CRITICAL_H_
-#define	_MACHINE_CRITICAL_H_
-
-__BEGIN_DECLS
-
-/*
- * Prototypes - see <arch>/<arch>/critical.c
- */
-void cpu_critical_fork_exit(void);
-
-#ifdef	__CC_SUPPORTS___INLINE
-
-/*
- *	cpu_critical_enter:
- *
- *	This routine is called from critical_enter() on the 0->1 transition
- *	of td_critnest, prior to it being incremented to 1.
- */
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-
-	td->td_md.md_savecrit = intr_disable();
-}
-
-/*
- *	cpu_critical_exit:
- *
- *	This routine is called from critical_exit() on a 1->0 transition
- *	of td_critnest, after it has been decremented to 0.  We are
- *	exiting the last critical section.
- */
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-
-	intr_restore(td->td_md.md_savecrit);
-}
-
-
-#else /* !__CC_SUPPORTS___INLINE */
-
-void cpu_critical_enter(struct thread *td)
-void cpu_critical_exit(struct thread *td)
-
-#endif	/* __CC_SUPPORTS___INLINE */
-
-__END_DECLS
-
-#endif /* !_MACHINE_CRITICAL_H_ */
-
diff --git a/sys/ia64/include/proc.h b/sys/ia64/include/proc.h
index 4d7f481df07e..6c8a1e10c7bd 100644
--- a/sys/ia64/include/proc.h
+++ b/sys/ia64/include/proc.h
@@ -30,7 +30,8 @@
 #define	_MACHINE_PROC_H_
 
 struct mdthread {
-	register_t	md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_intr;	/* (k) */
 };
 
 struct mdproc {
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 162d01967c74..310c3fcd9aea 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -72,7 +72,6 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 #include <vm/uma.h>
 
-#include <machine/critical.h>
 
 #ifndef _SYS_SYSPROTO_H_
 struct fork_args {
@@ -764,7 +763,6 @@ fork_exit(callout, arg, frame)
 
 	sched_lock.mtx_lock = (uintptr_t)td;
 	mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED);
-	cpu_critical_fork_exit();
 	CTR4(KTR_PROC, "fork_exit: new thread %p (kse %p, pid %d, %s)",
 		td, td->td_sched, p->p_pid, p->p_comm);
 
diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c
index 499794545cfd..1b4d432faf65 100644
--- a/sys/kern/kern_idle.c
+++ b/sys/kern/kern_idle.c
@@ -65,10 +65,6 @@ idle_setup(void *dummy)
 		error = kthread_create(idle_proc, NULL, &p,
 		    RFSTOPPED | RFHIGHPID, 0, "idle: cpu%d", pc->pc_cpuid);
 		pc->pc_idlethread = FIRST_THREAD_IN_PROC(p);
-		if (pc->pc_curthread == NULL) {
-			pc->pc_curthread = pc->pc_idlethread;
-			pc->pc_idlethread->td_critnest = 0;
-		}
 #else
 		error = kthread_create(idle_proc, NULL, &p,
 		    RFSTOPPED | RFHIGHPID, 0, "idle");
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index ed8f7ecae2e1..886ca5dd6ec5 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -586,7 +586,7 @@ _mtx_lock_spin(struct mtx *m, struct thread *td, int opts, const char *file,
 			break;
 
 		/* Give interrupts a chance while we spin. */
-		critical_exit();
+		spinlock_exit();
 		while (m->mtx_lock != MTX_UNOWNED) {
 			if (i++ < 10000000) {
 				cpu_spinwait();
@@ -605,7 +605,7 @@ _mtx_lock_spin(struct mtx *m, struct thread *td, int opts, const char *file,
 			}
 			cpu_spinwait();
 		}
-		critical_enter();
+		spinlock_enter();
 	}
 
 	if (LOCK_LOG_TEST(&m->mtx_object, opts))
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 0b3ecfdf0b78..feb512eb2e1c 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -63,7 +63,6 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/uma.h>
-#include <machine/critical.h>
 
 MALLOC_DEFINE(M_PGRP, "pgrp", "process group header");
 MALLOC_DEFINE(M_SESSION, "session", "session header");
diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c
index 24a29cbb22af..707a7f9980d5 100644
--- a/sys/kern/kern_switch.c
+++ b/sys/kern/kern_switch.c
@@ -105,7 +105,6 @@ __FBSDID("$FreeBSD$");
 #if defined(SMP) && (defined(__i386__) || defined(__amd64__))
 #include <sys/smp.h>
 #endif
-#include <machine/critical.h>
 #if defined(SMP) && defined(SCHED_4BSD)
 #include <sys/sysctl.h>
 #endif
@@ -581,8 +580,6 @@ critical_enter(void)
 	struct thread *td;
 
 	td = curthread;
-	if (td->td_critnest == 0)
-		cpu_critical_enter(td);
 	td->td_critnest++;
 	CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td,
 	    (long)td->td_proc->p_pid, td->td_proc->p_comm, td->td_critnest);
@@ -610,7 +607,6 @@ critical_exit(void)
 		}
 #endif
 		td->td_critnest = 0;
-		cpu_critical_exit(td);
 	} else {
 		td->td_critnest--;
 	}
diff --git a/sys/pc98/i386/machdep.c b/sys/pc98/i386/machdep.c
index 99ec50437608..92c0e1ed46bb 100644
--- a/sys/pc98/i386/machdep.c
+++ b/sys/pc98/i386/machdep.c
@@ -2280,6 +2280,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_flags = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_flags);
+}
+
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 static void f00f_hack(void *unused);
 SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL)
diff --git a/sys/pc98/include/critical.h b/sys/pc98/include/critical.h
deleted file mode 100644
index a3e709812e65..000000000000
--- a/sys/pc98/include/critical.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/*-
- * This file is in the public domain.
- */
-/* $FreeBSD$ */
-
-#include <i386/critical.h>
diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c
index 99ec50437608..92c0e1ed46bb 100644
--- a/sys/pc98/pc98/machdep.c
+++ b/sys/pc98/pc98/machdep.c
@@ -2280,6 +2280,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_flags = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_flags);
+}
+
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
 static void f00f_hack(void *unused);
 SYSINIT(f00f_hack, SI_SUB_INTRINSIC, SI_ORDER_FIRST, f00f_hack, NULL)
diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index 28cb3f9d2a60..3e0038a7d215 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -907,6 +907,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz)
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_msr = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_msr);
+}
+
 /*
  * kcopy(const void *src, void *dst, size_t len);
  *
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c
index 654a4be2d181..71c5c79289ac 100644
--- a/sys/powerpc/aim/vm_machdep.c
+++ b/sys/powerpc/aim/vm_machdep.c
@@ -154,6 +154,10 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 	pcb->pcb_lr = (register_t)fork_trampoline;
 	pcb->pcb_usr = kernel_pmap->pm_sr[USER_SR];
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_msr = PSL_KERNSET;
+
 	/*
  	 * Now cpu_switch() can schedule the new process.
 	 */
@@ -322,6 +326,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	pcb2->pcb_sp = (register_t)cf;
 	pcb2->pcb_lr = (register_t)fork_trampoline;
 	pcb2->pcb_usr = kernel_pmap->pm_sr[USER_SR];
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_msr = PSL_KERNSET;
 }
 
 void
diff --git a/sys/powerpc/include/critical.h b/sys/powerpc/include/critical.h
deleted file mode 100644
index 405bc57f0422..000000000000
--- a/sys/powerpc/include/critical.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef _MACHINE_CRITICAL_H_
-#define	_MACHINE_CRITICAL_H_
-
-__BEGIN_DECLS
-
-/*
- * Prototypes - see <arch>/<arch>/critical.c
- */
-void cpu_critical_fork_exit(void);
-
-#ifdef	__CC_SUPPORTS___INLINE
-
-/*
- *	cpu_critical_enter:
- *
- *	This routine is called from critical_enter() on the 0->1 transition
- *	of td_critnest, prior to it being incremented to 1.
- */
-
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-
-	td->td_md.md_savecrit = intr_disable();
-}
-
-/*
- *	cpu_critical_exit:
- *
- *	This routine is called from critical_exit() on a 1->0 transition
- *	of td_critnest, after it has been decremented to 0.  We are
- *	exiting the last critical section.
- */
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-
-	intr_restore(td->td_md.md_savecrit);
-}
-
-
-#else /* !__CC_SUPPORTS___INLINE */
-
-void cpu_critical_enter(struct thread *td);
-void cpu_critical_exit(struct thread *td);
-
-#endif	/* __CC_SUPPORTS___INLINE */
-
-__END_DECLS
-
-#endif /* !_MACHINE_CRITICAL_H_ */
-
diff --git a/sys/powerpc/include/proc.h b/sys/powerpc/include/proc.h
index e307b24ba1d9..c958fb76f0fb 100644
--- a/sys/powerpc/include/proc.h
+++ b/sys/powerpc/include/proc.h
@@ -39,7 +39,8 @@
  * Machine-dependent part of the proc structure
  */
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_msr;	/* (k) */
 };
 
 struct mdproc {
diff --git a/sys/powerpc/powerpc/critical.c b/sys/powerpc/powerpc/critical.c
deleted file mode 100644
index bd89a22a1882..000000000000
--- a/sys/powerpc/powerpc/critical.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/proc.h>
-#include <machine/critical.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- */
-void
-cpu_critical_fork_exit(void)
-{
-	struct thread *td = curthread;
-
-	td->td_md.md_savecrit = (mfmsr() | PSL_EE | PSL_RI);
-}
diff --git a/sys/powerpc/powerpc/machdep.c b/sys/powerpc/powerpc/machdep.c
index 28cb3f9d2a60..3e0038a7d215 100644
--- a/sys/powerpc/powerpc/machdep.c
+++ b/sys/powerpc/powerpc/machdep.c
@@ -907,6 +907,30 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz)
 
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0)
+		td->td_md.md_saved_msr = intr_disable();
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		intr_restore(td->td_md.md_saved_msr);
+}
+
 /*
  * kcopy(const void *src, void *dst, size_t len);
  *
diff --git a/sys/powerpc/powerpc/vm_machdep.c b/sys/powerpc/powerpc/vm_machdep.c
index 654a4be2d181..71c5c79289ac 100644
--- a/sys/powerpc/powerpc/vm_machdep.c
+++ b/sys/powerpc/powerpc/vm_machdep.c
@@ -154,6 +154,10 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 	pcb->pcb_lr = (register_t)fork_trampoline;
 	pcb->pcb_usr = kernel_pmap->pm_sr[USER_SR];
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_msr = PSL_KERNSET;
+
 	/*
  	 * Now cpu_switch() can schedule the new process.
 	 */
@@ -322,6 +326,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	pcb2->pcb_sp = (register_t)cf;
 	pcb2->pcb_lr = (register_t)fork_trampoline;
 	pcb2->pcb_usr = kernel_pmap->pm_sr[USER_SR];
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_msr = PSL_KERNSET;
 }
 
 void
diff --git a/sys/sparc64/include/critical.h b/sys/sparc64/include/critical.h
deleted file mode 100644
index 999a5fddc208..000000000000
--- a/sys/sparc64/include/critical.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * This file contains prototypes and high-level inlines related to
- * machine-level critical function support:
- *
- *	cpu_critical_enter()		- inlined
- *	cpu_critical_exit()		- inlined
- *	cpu_critical_fork_exit()	- prototyped
- *	related support functions residing
- *	in <arch>/<arch>/critical.c	- prototyped
- *
- * $FreeBSD$
- */
-
-#ifndef _MACHINE_CRITICAL_H_
-#define	_MACHINE_CRITICAL_H_
-
-__BEGIN_DECLS
-
-/*
- * Prototypes - see <arch>/<arch>/critical.c
- */
-void cpu_critical_fork_exit(void);
-
-#ifdef	__CC_SUPPORTS___INLINE
-
-/*
- *	cpu_critical_enter:
- *
- *	This routine is called from critical_enter() on the 0->1 transition
- *	of td_critnest, prior to it being incremented to 1.
- */
-static __inline void
-cpu_critical_enter(struct thread *td)
-{
-	critical_t pil;
-
-	pil = rdpr(pil);
-	wrpr(pil, 0, 14);
-	td->td_md.md_savecrit = pil;
-} 
- 
-
-/*
- *	cpu_critical_exit:
- *
- *	This routine is called from critical_exit() on a 1->0 transition
- *	of td_critnest, after it has been decremented to 0.  We are
- *	exiting the last critical section.
- */
-static __inline void
-cpu_critical_exit(struct thread *td)
-{
-
-	wrpr(pil, td->td_md.md_savecrit, 0);
-}
-
-#else /* !__CC_SUPPORTS___INLINE */
-
-void cpu_critical_enter(struct thread *td);
-void cpu_critical_exit(struct thread *td);
-
-#endif	/* __CC_SUPPORTS___INLINE */
-
-__END_DECLS
-
-#endif /* !_MACHINE_CRITICAL_H_ */
diff --git a/sys/sparc64/include/proc.h b/sys/sparc64/include/proc.h
index 3dbd1f408a58..bfd1268e433c 100644
--- a/sys/sparc64/include/proc.h
+++ b/sys/sparc64/include/proc.h
@@ -42,7 +42,8 @@ struct md_utrap {
 };
 
 struct mdthread {
-	register_t md_savecrit;
+	int	md_spinlock_count;	/* (k) */
+	register_t md_saved_pil;	/* (k) */
 };
 
 struct mdproc {
diff --git a/sys/sparc64/sparc64/critical.c b/sys/sparc64/sparc64/critical.c
deleted file mode 100644
index 622a6dc08a3c..000000000000
--- a/sys/sparc64/sparc64/critical.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*-
- * Copyright (c) 2002 Matthew Dillon.  All Rights Reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
- * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/pcpu.h>
-#include <sys/eventhandler.h>	/* XX */
-#include <sys/ktr.h>		/* XX */
-#include <sys/signalvar.h>
-#include <sys/sysproto.h>	/* XX */
-#include <sys/kernel.h>
-#include <sys/proc.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/sysctl.h>
-#include <sys/ucontext.h>
-#include <machine/critical.h>
-
-/*
- * cpu_critical_fork_exit() - cleanup after fork
- */
-void
-cpu_critical_fork_exit(void)
-{
-	struct thread *td;
-
-	td = curthread;
-	td->td_md.md_savecrit = 0;
-}
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index 1147a8e623d8..ccad87809bc5 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -232,6 +232,32 @@ cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 	}
 }
 
+void
+spinlock_enter(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (td->td_md.md_spinlock_count == 0) {
+		td->td_md.md_saved_pil = rdpr(pil);
+		wrpr(pil, 0, 14);
+	}
+	td->td_md.md_spinlock_count++;
+	critical_enter();
+}
+
+void
+spinlock_exit(void)
+{
+	struct thread *td;
+
+	td = curthread;
+	critical_exit();
+	td->td_md.md_spinlock_count--;
+	if (td->td_md.md_spinlock_count == 0)
+		wrpr(pil, td->td_md.md_saved_pil, 0);
+}
+
 unsigned
 tick_get_timecount(struct timecounter *tc)
 {
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index 9b56fc803dc6..b05e816a408a 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -309,9 +309,7 @@ cpu_mp_unleash(void *v)
 			continue;
 		KASSERT(pc->pc_idlethread != NULL,
 		    ("cpu_mp_unleash: idlethread"));
-		KASSERT(pc->pc_curthread == pc->pc_idlethread,
-		    ("cpu_mp_unleash: curthread"));
-	
+		pc->pc_curthread = pc->pc_idlethread;	
 		pc->pc_curpcb = pc->pc_curthread->td_pcb;
 		for (i = 0; i < PCPU_PAGES; i++) {
 			va = pc->pc_addr + i * PAGE_SIZE;
@@ -347,6 +345,7 @@ cpu_mp_bootstrap(struct pcpu *pc)
 	tick_start_ap();
 
 	smp_cpus++;
+	KASSERT(curthread != NULL, ("cpu_mp_bootstrap: curthread"));
 	PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid)));
 	printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
 
@@ -356,11 +355,11 @@ cpu_mp_bootstrap(struct pcpu *pc)
 	while (csa->csa_count != 0)
 		;
 
-	binuptime(PCPU_PTR(switchtime));
-	PCPU_SET(switchticks, ticks);
-
 	/* ok, now grab sched_lock and enter the scheduler */
 	mtx_lock_spin(&sched_lock);
+	spinlock_exit();
+	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchticks, ticks);
 	cpu_throw(NULL, choosethread());	/* doesn't return */
 }
 
diff --git a/sys/sparc64/sparc64/vm_machdep.c b/sys/sparc64/sparc64/vm_machdep.c
index 170cbea6bb8d..4c3ddf8dca04 100644
--- a/sys/sparc64/sparc64/vm_machdep.c
+++ b/sys/sparc64/sparc64/vm_machdep.c
@@ -170,6 +170,10 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	fr->fr_local[2] = (u_long)tf;
 	pcb->pcb_pc = (u_long)fork_trampoline - 8;
 	pcb->pcb_sp = (u_long)fr - SPOFF;
+
+	/* Setup to release sched_lock in fork_exit(). */
+	td->td_md.md_spinlock_count = 1;
+	td->td_md.md_saved_pil = 0;
 }
 
 void
@@ -281,6 +285,10 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 	pcb2->pcb_sp = (u_long)fp - SPOFF;
 	pcb2->pcb_pc = (u_long)fork_trampoline - 8;
 
+	/* Setup to release sched_lock in fork_exit(). */
+	td2->td_md.md_spinlock_count = 1;
+	td2->td_md.md_saved_pil = 0;
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 */
diff --git a/sys/sys/lock.h b/sys/sys/lock.h
index 34a22e29ca21..f8b168f9c399 100644
--- a/sys/sys/lock.h
+++ b/sys/sys/lock.h
@@ -196,6 +196,8 @@ extern struct lock_class lock_class_mtx_sleep;
 extern struct lock_class lock_class_mtx_spin;
 extern struct lock_class lock_class_sx;
 
+void	spinlock_enter(void);
+void	spinlock_exit(void);
 void	witness_init(struct lock_object *);
 void	witness_destroy(struct lock_object *);
 int	witness_defineorder(struct lock_object *, struct lock_object *);
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index 5fe31eb345f1..45eb504dae5a 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -167,7 +167,7 @@ void	_mtx_assert(struct mtx *m, int what, const char *file, int line);
 #define _get_spin_lock(mp, tid, opts, file, line) do {			\
 	struct thread *_tid = (tid);					\
 									\
-	critical_enter();						\
+	spinlock_enter();						\
 	if (!_obtain_lock((mp), _tid)) {				\
 		if ((mp)->mtx_lock == (uintptr_t)_tid)			\
 			(mp)->mtx_recurse++;				\
@@ -179,7 +179,7 @@ void	_mtx_assert(struct mtx *m, int what, const char *file, int line);
 #define _get_spin_lock(mp, tid, opts, file, line) do {			\
 	struct thread *_tid = (tid);					\
 									\
-	critical_enter();						\
+	spinlock_enter();						\
 	if ((mp)->mtx_lock == (uintptr_t)_tid)				\
 		(mp)->mtx_recurse++;					\
 	else {								\
@@ -207,8 +207,8 @@ void	_mtx_assert(struct mtx *m, int what, const char *file, int line);
  * Since spin locks are not _too_ common, inlining this code is not too big 
  * a deal.
  *
- * Since we always perform a critical_enter() when attempting to acquire a
- * spin lock, we need to always perform a matching critical_exit() when
+ * Since we always perform a spinlock_enter() when attempting to acquire a
+ * spin lock, we need to always perform a matching spinlock_exit() when
  * releasing a spin lock.  This includes the recursion cases.
  */
 #ifndef _rel_spin_lock
@@ -218,7 +218,7 @@ void	_mtx_assert(struct mtx *m, int what, const char *file, int line);
 		(mp)->mtx_recurse--;					\
 	else								\
 		_release_lock_quick((mp));				\
-	critical_exit();						\
+	spinlock_exit();						\
 } while (0)
 #else /* SMP */
 #define _rel_spin_lock(mp) do {						\
@@ -226,7 +226,7 @@ void	_mtx_assert(struct mtx *m, int what, const char *file, int line);
 		(mp)->mtx_recurse--;					\
 	else								\
 		(mp)->mtx_lock = MTX_UNOWNED;				\
-	critical_exit();						\
+	spinlock_exit();						\
 } while (0)
 #endif /* SMP */
 #endif