- Move manipulation of td_intr_nesting_level out of assembly interrupt

  vector stubs and into the C functions they call.
- Move disabling and EOIing of interrupt sources out of PIC driver entry
  points and into intr_execute_handlers().  Intr_execute_handlers() only
  disables a source for an interrupt if it is a stray interrupt or has
  threaded handlers.  Sources with fast handlers no longer disable (mask)
  the source while executing the handlers.
- Move the setting of clkintr_pending into intr_execute_handlers() and set
  the variable for any interrupt source with a vector of 0.  (Should only
  be true for IRQ 0.)  This fixes clkintr_pending in the NO_MIXED_MODE
  case.
- Implement lapic_eoi() and use it to implement ioapic_eoi_source().
- Rename atpic_sched_ithd() to atpic_handle_intr() since it is used to
  handle all atpic interrupts and not just threaded ones.

Inspired by:	peter's changes to amd64 in p4 (1)
Requested by:	bde (2)

10 files changed, 77 insertions, 99 deletions

diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s
index f1d252004c82..6c650db5821b 100644
--- a/sys/i386/i386/apic_vector.s
+++ b/sys/i386/i386/apic_vector.s
@@ -82,9 +82,7 @@ IDTVEC(vec_name) ;							\
 	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
 	mov	%ax, %fs ;						\
 	movl	lapic, %edx ;	/* pointer to local APIC */		\
-	movl	PCPU(CURTHREAD), %ebx ;					\
 	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
-	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
 	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
 	jz	2f ;							\
 	addl	$(32 * index),%eax ;					\
@@ -93,7 +91,6 @@ IDTVEC(vec_name) ;							\
 	pushl	%eax ;		/* pass the IRQ */			\
 	call	lapic_handle_intr ;					\
 	addl	$4, %esp ;	/* discard parameter */			\
-	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	doreti ;						\
 2:	movl	$-1, %eax ;	/* send a vector of -1 */		\
@@ -245,12 +242,9 @@ IDTVEC(hardclock)
 	movl	lapic, %edx
 	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
-	movl	PCPU(CURTHREAD),%ebx
-	incl	TD_INTR_NESTING_LEVEL(%ebx)
 	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_hardclock
 	addl	$4, %esp	/* XXX convert clockframe to trapframe */
-	decl	TD_INTR_NESTING_LEVEL(%ebx)
 	MEXITCOUNT
 	jmp	doreti
 
@@ -273,12 +267,9 @@ IDTVEC(statclock)
 
 	FAKE_MCOUNT(13*4(%esp))
 
-	movl	PCPU(CURTHREAD),%ebx
-	incl	TD_INTR_NESTING_LEVEL(%ebx)
 	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_statclock
 	addl	$4, %esp	/* XXX convert clockframe to trapframe */
-	decl	TD_INTR_NESTING_LEVEL(%ebx)
 	MEXITCOUNT
 	jmp	doreti
 
diff --git a/sys/i386/i386/intr_machdep.c b/sys/i386/i386/intr_machdep.c
index 2a4ece4d93e0..c8284f90c91b 100644
--- a/sys/i386/i386/intr_machdep.c
+++ b/sys/i386/i386/intr_machdep.c
@@ -49,6 +49,7 @@
 #include <sys/proc.h>
 #include <sys/syslog.h>
 #include <sys/systm.h>
+#include <machine/clock.h>
 #include <machine/intr_machdep.h>
 #ifdef DDB
 #include <ddb/ddb.h>
@@ -152,10 +153,14 @@ intr_remove_handler(void *cookie)
 void
 intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe)
 {
+	struct thread *td;
 	struct ithd *it;
 	struct intrhand *ih;
 	int error, vector;
 
+	td = curthread;
+	td->td_intr_nesting_level++;
+
 	/*
 	 * We count software interrupts when we process them.  The
 	 * code here follows previous practice, but there's an
@@ -165,18 +170,25 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe)
 	atomic_add_long(isrc->is_count, 1);
 	atomic_add_int(&cnt.v_intr, 1);
 
-	/*
-	 * Execute fast interrupt handlers directly.
-	 * To support clock handlers, if a handler registers
-	 * with a NULL argument, then we pass it a pointer to
-	 * a trapframe as its argument.
-	 */
 	it = isrc->is_ithread;
 	ih = TAILQ_FIRST(&it->it_handlers);
+
+	/*
+	 * XXX: We assume that IRQ 0 is only used for the ISA timer
+	 * device (clk).
+	 */
+	vector = isrc->is_pic->pic_vector(isrc);
+	if (vector == 0)
+		clkintr_pending = 1;
+
 	critical_enter();
-	if (ih == NULL)
-		error = EINVAL;
-	else if (ih->ih_flags & IH_FAST) {
+	if (ih != NULL && ih->ih_flags & IH_FAST) {
+		/*
+		 * Execute fast interrupt handlers directly.
+		 * To support clock handlers, if a handler registers
+		 * with a NULL argument, then we pass it a pointer to
+		 * a trapframe as its argument.
+		 */
 		TAILQ_FOREACH(ih, &it->it_handlers, ih_next) {
 			MPASS(ih->ih_flags & IH_FAST);
 			CTR3(KTR_INTR, "%s: executing handler %p(%p)",
@@ -188,13 +200,22 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe)
 			else
 				ih->ih_handler(ih->ih_argument);
 		}
-		isrc->is_pic->pic_enable_source(isrc);
+		isrc->is_pic->pic_eoi_source(isrc);
 		error = 0;
-	} else
-		error = ithread_schedule(it, !cold);
+	} else {
+		/*
+		 * For stray and threaded interrupts, we mask and EOI the
+		 * source.
+		 */
+		isrc->is_pic->pic_disable_source(isrc);
+		isrc->is_pic->pic_eoi_source(isrc);
+		if (ih == NULL)
+			error = EINVAL;
+		else
+			error = ithread_schedule(it, !cold);
+	}
 	critical_exit();
 	if (error == EINVAL) {
-		vector = isrc->is_pic->pic_vector(isrc);
 		atomic_add_long(isrc->is_straycount, 1);
 		if (*isrc->is_straycount < MAX_STRAY_LOG)
 			log(LOG_ERR, "stray irq%d\n", vector);
@@ -203,6 +224,7 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe)
 			    "too many stray irq %d's: not logging anymore\n",
 			    vector);
 	}
+	td->td_intr_nesting_level--;
 }
 
 void
diff --git a/sys/i386/i386/io_apic.c b/sys/i386/i386/io_apic.c
index c30513fd3168..4af70fa78d2c 100644
--- a/sys/i386/i386/io_apic.c
+++ b/sys/i386/i386/io_apic.c
@@ -201,8 +201,7 @@ static void
 ioapic_eoi_source(struct intsrc *isrc)
 {
 
-	TODO;
-	/* lapic_eoi(); */
+	lapic_eoi();
 }
 
 /*
diff --git a/sys/i386/i386/local_apic.c b/sys/i386/i386/local_apic.c
index 7fab550624bb..3fb7f76131f4 100644
--- a/sys/i386/i386/local_apic.c
+++ b/sys/i386/i386/local_apic.c
@@ -467,6 +467,13 @@ lapic_set_lvt_triggermode(u_int apic_id, u_int pin, u_char edgetrigger)
 }
 
 void
+lapic_eoi(void)
+{
+
+	lapic->eoi = 0;
+}
+
+void
 lapic_handle_intr(struct intrframe frame)
 {
 	struct intsrc *isrc;
@@ -474,8 +481,6 @@ lapic_handle_intr(struct intrframe frame)
 	if (frame.if_vec == -1)
 		panic("Couldn't get vector from ISR!");
 	isrc = intr_lookup_source(apic_idt_to_irq(frame.if_vec));
-	isrc->is_pic->pic_disable_source(isrc);
-	lapic->eoi = 0;
 	intr_execute_handlers(isrc, &frame);
 }
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 1c246e98407d..ff3ee05d3bc4 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -1047,12 +1047,16 @@ smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2)
 void
 forwarded_statclock(struct clockframe frame)
 {
+	struct thread *td;
 
 	CTR0(KTR_SMP, "forwarded_statclock");
+	td = curthread;
+	td->td_intr_nesting_level++;
 	if (profprocs != 0)
 		profclock(&frame);
 	if (pscnt == psdiv)
 		statclock(&frame);
+	td->td_intr_nesting_level--;
 }
 
 void
@@ -1080,9 +1084,13 @@ forward_statclock(void)
 void
 forwarded_hardclock(struct clockframe frame)
 {
+	struct thread *td;
 
 	CTR0(KTR_SMP, "forwarded_hardclock");
+	td = curthread;
+	td->td_intr_nesting_level++;
 	hardclock_process(&frame);
+	td->td_intr_nesting_level--;
 }
 
 void 
diff --git a/sys/i386/include/apicvar.h b/sys/i386/include/apicvar.h
index d43917226ed9..ea0521ebae6a 100644
--- a/sys/i386/include/apicvar.h
+++ b/sys/i386/include/apicvar.h
@@ -147,6 +147,7 @@ void	lapic_create(u_int apic_id, int boot_cpu);
 void	lapic_disable(void);
 void	lapic_dump(const char *str);
 void	lapic_enable_intr(u_int vector);
+void	lapic_eoi(void);
 int	lapic_id(void);
 void	lapic_init(uintptr_t addr);
 int	lapic_intr_pending(u_int vector);
diff --git a/sys/i386/include/clock.h b/sys/i386/include/clock.h
index f257809bc34c..1ee85620e621 100644
--- a/sys/i386/include/clock.h
+++ b/sys/i386/include/clock.h
@@ -15,6 +15,7 @@
  * XXX large parts of the driver and its interface are misplaced.
  */
 extern int	adjkerntz;
+extern int	clkintr_pending;
 extern int	disable_rtc_set;
 extern int	pscnt;
 extern int	psdiv;
diff --git a/sys/i386/isa/atpic.c b/sys/i386/isa/atpic.c
index 3dfb125587d9..0f17af3c326e 100644
--- a/sys/i386/isa/atpic.c
+++ b/sys/i386/isa/atpic.c
@@ -192,6 +192,10 @@ atpic_eoi_master(struct intsrc *isrc)
 #endif
 }
 
+/*
+ * The data sheet says no auto-EOI on slave, but it sometimes works.
+ * So, if AUTO_EOI_2 is enabled, we use it.
+ */
 static void
 atpic_eoi_slave(struct intsrc *isrc)
 {
@@ -327,7 +331,7 @@ atpic_init(void *dummy __unused)
 SYSINIT(atpic_init, SI_SUB_INTR, SI_ORDER_SECOND + 1, atpic_init, NULL)
 
 void
-atpic_sched_ithd(struct intrframe iframe)
+atpic_handle_intr(struct intrframe iframe)
 {
 	struct intsrc *isrc;
 
diff --git a/sys/i386/isa/atpic_vector.s b/sys/i386/isa/atpic_vector.s
index 2da86e0dc21d..d01be396d24d 100644
--- a/sys/i386/isa/atpic_vector.s
+++ b/sys/i386/isa/atpic_vector.s
@@ -52,50 +52,10 @@
 
 #include "assym.s"
 
-#define	IRQ_BIT(irq_num)	(1 << ((irq_num) % 8))
-#define	IRQ_BYTE(irq_num)	((irq_num) >> 3)
-
-#ifdef AUTO_EOI_1
-
-#define	ENABLE_ICU1		/* use auto-EOI to reduce i/o */
-#define	OUTB_ICU1
-
-#else
-
-#define	ENABLE_ICU1							\
-	movb	$ICU_EOI,%al ;	/* as soon as possible send EOI ... */	\
-	OUTB_ICU1		/* ... to clear in service bit */
-
-#define	OUTB_ICU1							\
-	outb	%al,$IO_ICU1
-
-#endif
-
-#ifdef AUTO_EOI_2
-/*
- * The data sheet says no auto-EOI on slave, but it sometimes works.
- */
-#define	ENABLE_ICU1_AND_2	ENABLE_ICU1
-
-#else
-
-#define	ENABLE_ICU1_AND_2						\
-	movb	$ICU_EOI,%al ;	/* as above */				\
-	outb	%al,$IO_ICU2 ;	/* but do second icu first ... */	\
-	OUTB_ICU1		/* ... then first icu (if !AUTO_EOI_1) */
-
-#endif
-
 /*
  * Macros for interrupt interrupt entry, call to handler, and exit.
- *
- * XXX Most of the parameters here are obsolete.  Fix this when we're
- * done.
- * XXX we really shouldn't return via doreti if we just schedule the
- * interrupt handler and don't run anything.  We could just do an
- * iret.  FIXME.
  */
-#define	INTR(irq_num, vec_name, icu, enable_icus, maybe_extra_ipending) \
+#define	INTR(irq_num, vec_name) \
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
 IDTVEC(vec_name) ;							\
@@ -111,42 +71,29 @@ IDTVEC(vec_name) ;							\
 	mov	$KPSEL,%ax ;						\
 	mov	%ax,%fs ;						\
 ;									\
-	maybe_extra_ipending ;						\
-	movb	imen + IRQ_BYTE(irq_num),%al ;				\
-	orb	$IRQ_BIT(irq_num),%al ;					\
-	movb	%al,imen + IRQ_BYTE(irq_num) ;				\
-	outb	%al,$icu+ICU_IMR_OFFSET ;				\
-	enable_icus ;							\
-;									\
-	movl	PCPU(CURTHREAD),%ebx ;					\
-	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-;									\
 	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX late to avoid double count */ \
 	pushl	$irq_num; 	/* pass the IRQ */			\
-	call	atpic_sched_ithd ;					\
+	call	atpic_handle_intr ;					\
 	addl	$4, %esp ;	/* discard the parameter */		\
 ;									\
-	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	doreti
 
 MCOUNT_LABEL(bintr)
-#define	CLKINTR_PENDING	movl $1,CNAME(clkintr_pending)
-/* Threaded interrupts */
-	INTR(0,atpic_intr0, IO_ICU1, ENABLE_ICU1, CLKINTR_PENDING)
-	INTR(1,atpic_intr1, IO_ICU1, ENABLE_ICU1,)
-	INTR(2,atpic_intr2, IO_ICU1, ENABLE_ICU1,)
-	INTR(3,atpic_intr3, IO_ICU1, ENABLE_ICU1,)
-	INTR(4,atpic_intr4, IO_ICU1, ENABLE_ICU1,)
-	INTR(5,atpic_intr5, IO_ICU1, ENABLE_ICU1,)
-	INTR(6,atpic_intr6, IO_ICU1, ENABLE_ICU1,)
-	INTR(7,atpic_intr7, IO_ICU1, ENABLE_ICU1,)
-	INTR(8,atpic_intr8, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(9,atpic_intr9, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(10,atpic_intr10, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(11,atpic_intr11, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(12,atpic_intr12, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(13,atpic_intr13, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(14,atpic_intr14, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(15,atpic_intr15, IO_ICU2, ENABLE_ICU1_AND_2,)
+	INTR(0, atpic_intr0)
+	INTR(1, atpic_intr1)
+	INTR(2, atpic_intr2)
+	INTR(3, atpic_intr3)
+	INTR(4, atpic_intr4)
+	INTR(5, atpic_intr5)
+	INTR(6, atpic_intr6)
+	INTR(7, atpic_intr7)
+	INTR(8, atpic_intr8)
+	INTR(9, atpic_intr9)
+	INTR(10, atpic_intr10)
+	INTR(11, atpic_intr11)
+	INTR(12, atpic_intr12)
+	INTR(13, atpic_intr13)
+	INTR(14, atpic_intr14)
+	INTR(15, atpic_intr15)
 MCOUNT_LABEL(eintr)
diff --git a/sys/i386/isa/icu.h b/sys/i386/isa/icu.h
index 8958f4a93857..512afefc99d4 100644
--- a/sys/i386/isa/icu.h
+++ b/sys/i386/isa/icu.h
@@ -104,7 +104,7 @@
 #define	HWI_MASK	0xffff		/* bits for h/w interrupts */
 
 #ifndef LOCORE
-void	atpic_sched_ithd(struct intrframe iframe);
+void	atpic_handle_intr(struct intrframe iframe);
 void	atpic_startup(void);
 #endif