Make intr_nesting_level per-process, rather than per-cpu. Setup

interrupt threads to run with it always >= 1, so that malloc can
detect M_WAITOK from "interrupt" context.  This is also necessary
in order to context switch from sched_ithd() directly.

Reviewed By:	peter

7 files changed, 21 insertions, 17 deletions

diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s
index 4ebf35d3320b..a2cdb47bf942 100644
--- a/sys/i386/i386/apic_vector.s
+++ b/sys/i386/i386/apic_vector.s
@@ -48,7 +48,8 @@ IDTVEC(vec_name) ;							\
 	movl	$KPSEL,%eax ;						\
 	mov	%ax,%fs ;						\
 	FAKE_MCOUNT(13*4(%esp)) ;					\
-	incb	PCPU(INTR_NESTING_LEVEL) ;				\
+	movl	PCPU(CURPROC),%ebx ;					\
+	incl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	pushl	_intr_unit + (irq_num) * 4 ;				\
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	addl	$4, %esp ;						\
@@ -58,6 +59,7 @@ IDTVEC(vec_name) ;							\
 	movl	_intr_countp + (irq_num) * 4, %eax ;			\
 	lock ; 								\
 	incl	(%eax) ;						\
+	decl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
@@ -150,7 +152,8 @@ IDTVEC(vec_name) ;							\
 	MASK_LEVEL_IRQ(irq_num) ;					\
 	EOI_IRQ(irq_num) ;						\
 0: ;									\
-	incb	PCPU(INTR_NESTING_LEVEL) ;				\
+	movl	PCPU(CURPROC),%ebx ;					\
+	incl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 ;	 								\
   /* entry point used by doreti_unpend for HWIs. */			\
 __CONCAT(Xresume,irq_num): ;						\
@@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ;						\
 	call	_sched_ithd ;						\
 	addl	$4, %esp ;		/* discard the parameter */	\
 ;									\
+	decl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
@@ -301,7 +305,8 @@ _Xcpuast:
 	FAKE_MCOUNT(13*4(%esp))
 
 	orl	$AST_PENDING, PCPU(ASTPENDING)	/* XXX */
-	incb	PCPU(INTR_NESTING_LEVEL)
+	movl	PCPU(CURPROC),%ebx
+	incl	P_INTR_NESTING_LEVEL(%ebx)
 	sti
 	
 	movl	PCPU(CPUID), %eax
@@ -316,6 +321,7 @@ _Xcpuast:
 2:		
 	lock
 	incl	CNAME(cpuast_cnt)
+	decl	P_INTR_NESTING_LEVEL(%ebx)
 	MEXITCOUNT
 	jmp	_doreti
 1:
@@ -323,6 +329,7 @@ _Xcpuast:
 	POP_FRAME
 	iret			
 
+#if 0
 
 /*
  *	 Executed by a CPU when it receives an XFORWARD_IRQ IPI.
@@ -360,7 +367,6 @@ _Xforward_irq:
 	POP_FRAME
 	iret
 
-#if 0
 /*
  * 
  */
diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index 603b4be6a236..aafbf0b8c5c9 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -177,7 +177,6 @@ IDTVEC(fpu)
 	call	_npx_intr
 
 	addl	$4,%esp
-	incb	PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 #else	/* DEV_NPX */
@@ -216,7 +215,6 @@ calltrap:
 	/*
 	 * Return via _doreti to handle ASTs.
 	 */
-	incb	PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 
@@ -257,7 +255,6 @@ IDTVEC(syscall)
 	cli				/* atomic astpending access */
 	cmpl    $0,PCPU(ASTPENDING)	/* AST pending? */
 	je	doreti_syscall_ret	/* no, get out of here */
-	movb	$1,PCPU(INTR_NESTING_LEVEL)
 	jmp	_doreti
 
 /*
@@ -289,7 +286,6 @@ IDTVEC(int0x80_syscall)
 	cli				/* atomic astpending access */
 	cmpl    $0,PCPU(ASTPENDING)	/* AST pending? */
 	je	doreti_syscall_ret	/* no, get out of here */
-	movb	$1,PCPU(INTR_NESTING_LEVEL)
 	jmp	_doreti
 
 ENTRY(fork_trampoline)
@@ -323,7 +319,6 @@ ENTRY(fork_trampoline)
 	/*
 	 * Return via _doreti to handle ASTs.
 	 */
-	movb	$1,PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 
diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c
index 8b9ca330d0a4..c307665c1b90 100644
--- a/sys/i386/i386/genassym.c
+++ b/sys/i386/i386/genassym.c
@@ -80,6 +80,7 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
 ASSYM(P_ADDR, offsetof(struct proc, p_addr));
+ASSYM(P_INTR_NESTING_LEVEL, offsetof(struct proc, p_intr_nesting_level));
 ASSYM(P_STAT, offsetof(struct proc, p_stat));
 ASSYM(P_WCHAN, offsetof(struct proc, p_wchan));
 
@@ -182,7 +183,6 @@ ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt));
 ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending));
 ASSYM(AST_PENDING, AST_PENDING);
 ASSYM(AST_RESCHED, AST_RESCHED);
-ASSYM(GD_INTR_NESTING_LEVEL, offsetof(struct globaldata, gd_intr_nesting_level));
 
 #ifdef USER_LDT
 ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt));
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index bf560cd07efa..966d3f897d52 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -581,10 +581,12 @@ mp_enable(u_int boot_addr)
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	
+
+#if 0	
 	/* install an inter-CPU IPI for interrupt forwarding */
 	setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#endif
 
 	/* install an inter-CPU IPI for CPU stop/restart */
 	setidt(XCPUSTOP_OFFSET, Xcpustop,
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index bf560cd07efa..966d3f897d52 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -581,10 +581,12 @@ mp_enable(u_int boot_addr)
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	
+
+#if 0	
 	/* install an inter-CPU IPI for interrupt forwarding */
 	setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#endif
 
 	/* install an inter-CPU IPI for CPU stop/restart */
 	setidt(XCPUSTOP_OFFSET, Xcpustop,
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index 6ee3e6d77708..0170dbef49d7 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -480,7 +480,7 @@ restart:
 			if (in_vm86call)
 				break;
 
-			if (PCPU_GET(intr_nesting_level) != 0)
+			if (p->p_intr_nesting_level != 0)
 				break;
 
 			/*
@@ -687,7 +687,7 @@ trap_pfault(frame, usermode, eva)
 
 		if (p == NULL ||
 		    (!usermode && va < VM_MAXUSER_ADDRESS &&
-		     (PCPU_GET(intr_nesting_level) != 0 ||
+		     (p->p_intr_nesting_level != 0 ||
 		      PCPU_GET(curpcb) == NULL ||
 		      PCPU_GET(curpcb)->pcb_onfault == NULL))) {
 			trap_fatal(frame, eva);
@@ -751,7 +751,7 @@ trap_pfault(frame, usermode, eva)
 		return (0);
 nogo:
 	if (!usermode) {
-		if (PCPU_GET(intr_nesting_level) == 0 &&
+		if (p->p_intr_nesting_level == 0 &&
 		    PCPU_GET(curpcb) != NULL &&
 		    PCPU_GET(curpcb)->pcb_onfault != NULL) {
 			frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault;
@@ -858,7 +858,7 @@ trap_pfault(frame, usermode, eva)
 		return (0);
 nogo:
 	if (!usermode) {
-		if (PCPU_GET(intr_nesting_level) == 0 &&
+		if (p->p_intr_nesting_level == 0 &&
 		    PCPU_GET(curpcb) != NULL &&
 		    PCPU_GET(curpcb)->pcb_onfault != NULL) {
 			frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault;
diff --git a/sys/i386/i386/vm86bios.s b/sys/i386/i386/vm86bios.s
index a71dd11aabde..5f3013c4b991 100644
--- a/sys/i386/i386/vm86bios.s
+++ b/sys/i386/i386/vm86bios.s
@@ -131,7 +131,6 @@ ENTRY(vm86_bioscall)
 	/*
 	 * Return via _doreti
 	 */
-	incb	PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti