path: root/sys/powerpc/booke/trap_subr.S

/*-
 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
 * Copyright (C) 2006 Juniper Networks, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD$
 */
/*-
 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
 * Copyright (C) 1995, 1996 TooLs GmbH.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by TooLs GmbH.
 * 4. The name of TooLs GmbH may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *	from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
 */

/*
 * NOTICE: This is not a standalone file.  to use it, #include it in
 * your port's locore.S, like so:
 *
 *	#include <powerpc/booke/trap_subr.S>
 */

/*
 * SPRG usage notes
 *
 * SPRG0 - pcpu pointer
 * SPRG1 - all interrupts except TLB miss, critical, machine check
 * SPRG2 - critical
 * SPRG3 - machine check
 * SPRG4-6 - scratch
 *
 */

/* Get the per-CPU data structure */
#define GET_CPUINFO(r) mfsprg0 r

#define RES_GRANULE	64
#define RES_LOCK	0	/* offset to the 'lock' word */
#ifdef __powerpc64__
#define RES_RECURSE	8	/* offset to the 'recurse' word */
#else
#define RES_RECURSE	4	/* offset to the 'recurse' word */
#endif

/*
 * Standard interrupt prolog
 *
 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
 *           SRR0-1, CSRR0-1, MCSRR0-1
 *
 * 1. saves in the given savearea:
 *   - R30-31
 *   - DEAR, ESR
 *   - xSRR0-1
 *
 * 2. saves CR -> R30
 *
 * 3. switches to kstack if needed
 *
 * 4. notes:
 *   - R31 can be used as scratch register until a new frame is layed on
 *     the stack with FRAME_SETUP
 *
 *   - potential TLB miss: NO. Saveareas are always acessible via TLB1 
 *     permanent entries, and within this prolog we do not dereference any
 *     locations potentially not in the TLB
 */
#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
	mtspr	sprg_sp, %r1;		/* Save SP */			\
	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
	STORE	%r30, (savearea+CPUSAVE_R30)(%r1);			\
	STORE	%r31, (savearea+CPUSAVE_R31)(%r1); 			\
	mfspr	%r30, SPR_DEAR;						\
	mfspr	%r31, SPR_ESR;						\
	STORE	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); 		\
	STORE	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); 		\
	mfspr	%r30, isrr0;						\
	mfspr	%r31, isrr1;	 	/* MSR at interrupt time */	\
	STORE	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
	STORE	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
	isync;			 					\
	mfspr	%r1, sprg_sp;	 	/* Restore SP */		\
	mfcr	%r30;		 	/* Save CR */			\
	/* switch to per-thread kstack if intr taken in user mode */	\
	mtcr	%r31;			/* MSR at interrupt time  */	\
	bf	17, 1f;							\
	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
	LOAD	%r1, PC_CURPCB(%r1); 	/* Per-thread kernel stack */	\
1:

#define	STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
	mtspr	sprg_sp, %r1;		/* Save SP */			\
	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
	STORE	%r30, (savearea+CPUSAVE_R30)(%r1);			\
	STORE	%r31, (savearea+CPUSAVE_R31)(%r1);			\
	mfspr	%r30, SPR_DEAR;						\
	mfspr	%r31, SPR_ESR;						\
	STORE	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);		\
	STORE	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);		\
	mfspr	%r30, isrr0;						\
	mfspr	%r31, isrr1;		/* MSR at interrupt time */	\
	STORE	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
	STORE	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
	mfspr	%r30, SPR_SRR0;						\
	mfspr	%r31, SPR_SRR1;		/* MSR at interrupt time */	\
	STORE	%r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1);		\
	STORE	%r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1);		\
	isync;								\
	mfspr	%r1, sprg_sp;		/* Restore SP */		\
	mfcr	%r30;			/* Save CR */			\
	/* switch to per-thread kstack if intr taken in user mode */	\
	mtcr	%r31;			/* MSR at interrupt time  */	\
	bf	17, 1f;							\
	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
	LOAD	%r1, PC_CURPCB(%r1);	/* Per-thread kernel stack */	\
1:

/*
 * FRAME_SETUP assumes:
 *	SPRG{1-3}	SP at the time interrupt occured
 *	savearea	r30-r31, DEAR, ESR, xSRR0-1
 *	r30		CR
 *	r31		scratch
 *	r1		kernel stack
 *
 * sprg_sp - SPRG reg containing SP at the time interrupt occured
 * savearea - temp save
 * exc - exception number (EXC_xxx)
 *
 * 1. sets a new frame
 * 2. saves in the frame:
 *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
 *   - R3-31 (R30-31 first restored from savearea)
 *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
 *
 * Notes:
 * - potential TLB miss: YES, since we make dereferences to kstack, which
 *   can happen not covered (we can have up to two DTLB misses if fortunate
 *   enough i.e. when kstack crosses page boundary and both pages are
 *   untranslated)
 */
#ifdef __powerpc64__
#define SAVE_REGS(r)							\
	std	%r3, FRAME_3+CALLSIZE(r);				\
	std	%r4, FRAME_4+CALLSIZE(r);				\
	std	%r5, FRAME_5+CALLSIZE(r);				\
	std	%r6, FRAME_6+CALLSIZE(r);				\
	std	%r7, FRAME_7+CALLSIZE(r);				\
	std	%r8, FRAME_8+CALLSIZE(r);				\
	std	%r9, FRAME_9+CALLSIZE(r);				\
	std	%r10, FRAME_10+CALLSIZE(r);				\
	std	%r11, FRAME_11+CALLSIZE(r);				\
	std	%r12, FRAME_12+CALLSIZE(r);				\
	std	%r13, FRAME_13+CALLSIZE(r);				\
	std	%r14, FRAME_14+CALLSIZE(r);				\
	std	%r15, FRAME_15+CALLSIZE(r);				\
	std	%r16, FRAME_16+CALLSIZE(r);				\
	std	%r17, FRAME_17+CALLSIZE(r);				\
	std	%r18, FRAME_18+CALLSIZE(r);				\
	std	%r19, FRAME_19+CALLSIZE(r);				\
	std	%r20, FRAME_20+CALLSIZE(r);				\
	std	%r21, FRAME_21+CALLSIZE(r);				\
	std	%r22, FRAME_22+CALLSIZE(r);				\
	std	%r23, FRAME_23+CALLSIZE(r);				\
	std	%r24, FRAME_24+CALLSIZE(r);				\
	std	%r25, FRAME_25+CALLSIZE(r);				\
	std	%r26, FRAME_26+CALLSIZE(r);				\
	std	%r27, FRAME_27+CALLSIZE(r);				\
	std	%r28, FRAME_28+CALLSIZE(r);				\
	std	%r29, FRAME_29+CALLSIZE(r);				\
	std	%r30, FRAME_30+CALLSIZE(r);				\
	std	%r31, FRAME_31+CALLSIZE(r)
#define LD_REGS(r)							\
	ld	%r3, FRAME_3+CALLSIZE(r);				\
	ld	%r4, FRAME_4+CALLSIZE(r);				\
	ld	%r5, FRAME_5+CALLSIZE(r);				\
	ld	%r6, FRAME_6+CALLSIZE(r);				\
	ld	%r7, FRAME_7+CALLSIZE(r);				\
	ld	%r8, FRAME_8+CALLSIZE(r);				\
	ld	%r9, FRAME_9+CALLSIZE(r);				\
	ld	%r10, FRAME_10+CALLSIZE(r);				\
	ld	%r11, FRAME_11+CALLSIZE(r);				\
	ld	%r12, FRAME_12+CALLSIZE(r);				\
	ld	%r13, FRAME_13+CALLSIZE(r);				\
	ld	%r14, FRAME_14+CALLSIZE(r);				\
	ld	%r15, FRAME_15+CALLSIZE(r);				\
	ld	%r16, FRAME_16+CALLSIZE(r);				\
	ld	%r17, FRAME_17+CALLSIZE(r);				\
	ld	%r18, FRAME_18+CALLSIZE(r);				\
	ld	%r19, FRAME_19+CALLSIZE(r);				\
	ld	%r20, FRAME_20+CALLSIZE(r);				\
	ld	%r21, FRAME_21+CALLSIZE(r);				\
	ld	%r22, FRAME_22+CALLSIZE(r);				\
	ld	%r23, FRAME_23+CALLSIZE(r);				\
	ld	%r24, FRAME_24+CALLSIZE(r);				\
	ld	%r25, FRAME_25+CALLSIZE(r);				\
	ld	%r26, FRAME_26+CALLSIZE(r);				\
	ld	%r27, FRAME_27+CALLSIZE(r);				\
	ld	%r28, FRAME_28+CALLSIZE(r);				\
	ld	%r29, FRAME_29+CALLSIZE(r);				\
	ld	%r30, FRAME_30+CALLSIZE(r);				\
	ld	%r31, FRAME_31+CALLSIZE(r)
#else
#define SAVE_REGS(r)							\
	stmw	%r3,  FRAME_3+CALLSIZE(r)
#define LD_REGS(r)							\
	lmw	%r3,  FRAME_3+CALLSIZE(r)
#endif
#define	FRAME_SETUP(sprg_sp, savearea, exc)				\
	mfspr	%r31, sprg_sp;		/* get saved SP */		\
	/* establish a new stack frame and put everything on it */	\
	STU	%r31, -(FRAMELEN+REDZONE)(%r1);				\
	STORE	%r0, FRAME_0+CALLSIZE(%r1);	/* save r0 in the trapframe */	\
	STORE	%r31, FRAME_1+CALLSIZE(%r1);	/* save SP   "     " */	\
	STORE	%r2, FRAME_2+CALLSIZE(%r1);	/* save r2   "     " */	\
	mflr	%r31;		 					\
	STORE	%r31, FRAME_LR+CALLSIZE(%r1);	/* save LR   "     " */	\
	STORE	%r30, FRAME_CR+CALLSIZE(%r1);	/* save CR   "     " */	\
	GET_CPUINFO(%r2);						\
	LOAD	%r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */	\
	LOAD	%r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */	\
	/* save R3-31 */						\
	SAVE_REGS(%r1);							\
	/* save DEAR, ESR */						\
	LOAD	%r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);		\
	LOAD	%r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);		\
	STORE	%r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1);			\
	STORE	%r29, FRAME_BOOKE_ESR+CALLSIZE(%r1);			\
	/* save XER, CTR, exc number */					\
	mfxer	%r3;							\
	mfctr	%r4;							\
	STORE	%r3, FRAME_XER+CALLSIZE(%r1);				\
	STORE	%r4, FRAME_CTR+CALLSIZE(%r1);				\
	li	%r5, exc;						\
	STORE	%r5, FRAME_EXC+CALLSIZE(%r1);				\
	/* save DBCR0 */						\
	mfspr	%r3, SPR_DBCR0;						\
	STORE	%r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);			\
	/* save xSSR0-1 */						\
	LOAD	%r30, (savearea+CPUSAVE_SRR0)(%r2);			\
	LOAD	%r31, (savearea+CPUSAVE_SRR1)(%r2);			\
	STORE	%r30, FRAME_SRR0+CALLSIZE(%r1);				\
	STORE	%r31, FRAME_SRR1+CALLSIZE(%r1);				\
	LOAD	THREAD_REG, PC_CURTHREAD(%r2);				\

/*
 *
 * isrr0-1 - save restore registers to restore CPU state to (may be
 *           SRR0-1, CSRR0-1, MCSRR0-1
 *
 * Notes:
 *  - potential TLB miss: YES. The deref'd kstack may be not covered
 */
#define	FRAME_LEAVE(isrr0, isrr1)					\
	wrteei 0;							\
	/* restore CTR, XER, LR, CR */					\
	LOAD	%r4, FRAME_CTR+CALLSIZE(%r1);				\
	LOAD	%r5, FRAME_XER+CALLSIZE(%r1);				\
	LOAD	%r6, FRAME_LR+CALLSIZE(%r1);				\
	LOAD	%r7, FRAME_CR+CALLSIZE(%r1);				\
	mtctr	%r4;							\
	mtxer	%r5;							\
	mtlr	%r6;							\
	mtcr	%r7;							\
	/* restore DBCR0 */						\
	LOAD	%r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);			\
	mtspr	SPR_DBCR0, %r4;						\
	/* restore xSRR0-1 */						\
	LOAD	%r30, FRAME_SRR0+CALLSIZE(%r1);				\
	LOAD	%r31, FRAME_SRR1+CALLSIZE(%r1);				\
	mtspr	isrr0, %r30;						\
	mtspr	isrr1, %r31;						\
	/* restore R2-31, SP */						\
	LD_REGS(%r1);							\
	LOAD	%r2, FRAME_2+CALLSIZE(%r1);				\
	LOAD	%r0, FRAME_0+CALLSIZE(%r1);				\
	LOAD	%r1, FRAME_1+CALLSIZE(%r1);				\
	isync

/*
 * TLB miss prolog
 *
 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
 *
 * Notes:
 *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
 *    miss within the TLB prolog itself!
 *  - TLBSAVE is always translated
 */
#ifdef __powerpc64__
#define	TLB_SAVE_REGS(br)						\
	std	%r20, (TLBSAVE_BOOKE_R20)(br);				\
	std	%r21, (TLBSAVE_BOOKE_R21)(br);				\
	std	%r22, (TLBSAVE_BOOKE_R22)(br);				\
	std	%r23, (TLBSAVE_BOOKE_R23)(br);				\
	std	%r24, (TLBSAVE_BOOKE_R24)(br);				\
	std	%r25, (TLBSAVE_BOOKE_R25)(br);				\
	std	%r26, (TLBSAVE_BOOKE_R26)(br);				\
	std	%r27, (TLBSAVE_BOOKE_R27)(br);				\
	std	%r28, (TLBSAVE_BOOKE_R28)(br);				\
	std	%r29, (TLBSAVE_BOOKE_R29)(br);				\
	std	%r30, (TLBSAVE_BOOKE_R30)(br);				\
	std	%r31, (TLBSAVE_BOOKE_R31)(br);				
#define	TLB_RESTORE_REGS(br)						\
	ld	%r20, (TLBSAVE_BOOKE_R20)(br);				\
	ld	%r21, (TLBSAVE_BOOKE_R21)(br);				\
	ld	%r22, (TLBSAVE_BOOKE_R22)(br);				\
	ld	%r23, (TLBSAVE_BOOKE_R23)(br);				\
	ld	%r24, (TLBSAVE_BOOKE_R24)(br);				\
	ld	%r25, (TLBSAVE_BOOKE_R25)(br);				\
	ld	%r26, (TLBSAVE_BOOKE_R26)(br);				\
	ld	%r27, (TLBSAVE_BOOKE_R27)(br);				\
	ld	%r28, (TLBSAVE_BOOKE_R28)(br);				\
	ld	%r29, (TLBSAVE_BOOKE_R29)(br);				\
	ld	%r30, (TLBSAVE_BOOKE_R30)(br);				\
	ld	%r31, (TLBSAVE_BOOKE_R31)(br);				
#define TLB_NEST(outr,inr)						\
	rlwinm	outr, inr, 7, 23, 24;	/* 8 x TLBSAVE_LEN */
#else
#define TLB_SAVE_REGS(br)						\
	stmw	%r20, TLBSAVE_BOOKE_R20(br)
#define TLB_RESTORE_REGS(br)						\
	lmw	%r20, TLBSAVE_BOOKE_R20(br)
#define TLB_NEST(outr,inr)						\
	rlwinm	outr, inr, 6, 24, 25;	/* 4 x TLBSAVE_LEN */
#endif
#define TLB_PROLOG							\
	mtspr	SPR_SPRG4, %r1;			/* Save SP */		\
	mtspr	SPR_SPRG5, %r28;					\
	mtspr	SPR_SPRG6, %r29;					\
	/* calculate TLB nesting level and TLBSAVE instance address */	\
	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
	LOAD	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
	TLB_NEST(%r29,%r28);						\
	addi	%r28, %r28, 1;						\
	STORE	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l; 			\
	add	%r1, %r1, %r29;		/* current TLBSAVE ptr */	\
									\
	/* save R20-31 */						\
	mfspr	%r28, SPR_SPRG5;		 			\
	mfspr	%r29, SPR_SPRG6;					\
	TLB_SAVE_REGS(%r1);			\
	/* save LR, CR */						\
	mflr	%r30;		 					\
	mfcr	%r31;							\
	STORE	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
	STORE	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
	/* save SRR0-1 */						\
	mfsrr0	%r30;		/* execution addr at interrupt time */	\
	mfsrr1	%r31;		/* MSR at interrupt time*/		\
	STORE	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);	/* save SRR0 */	\
	STORE	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);	/* save SRR1 */	\
	isync;								\
	mfspr	%r1, SPR_SPRG4

/*
 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
 *
 * same notes as for the TLB_PROLOG
 */
#define TLB_RESTORE							\
	mtspr	SPR_SPRG4, %r1;			/* Save SP */		\
	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
	/* calculate TLB nesting level and TLBSAVE instance addr */	\
	LOAD	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
	subi	%r28, %r28, 1;						\
	STORE	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
	TLB_NEST(%r29,%r28);						\
	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l;				\
	add	%r1, %r1, %r29;						\
									\
	/* restore LR, CR */						\
	LOAD	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
	LOAD	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
	mtlr	%r30;							\
	mtcr	%r31;							\
	/* restore SRR0-1 */						\
	LOAD	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);			\
	LOAD	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);			\
	mtsrr0	%r30;							\
	mtsrr1	%r31;							\
	/* restore R20-31 */						\
	TLB_RESTORE_REGS(%r1);						\
	mfspr	%r1, SPR_SPRG4

#ifdef SMP
#define TLB_LOCK							\
	GET_CPUINFO(%r20);						\
	LOAD	%r21, PC_CURTHREAD(%r20);				\
	LOAD	%r22, PC_BOOKE_TLB_LOCK(%r20);				\
									\
1:	LOADX	%r23, 0, %r22;						\
	CMPI	%r23, TLB_UNLOCKED;					\
	beq	2f;							\
									\
	/* check if this is recursion */				\
	CMPL	cr0, %r21, %r23;					\
	bne-	1b;							\
									\
2:	/* try to acquire lock */					\
	STOREX	%r21, 0, %r22;						\
	bne-	1b;							\
									\
	/* got it, update recursion counter */				\
	lwz	%r21, RES_RECURSE(%r22);				\
	addi	%r21, %r21, 1;						\
	stw	%r21, RES_RECURSE(%r22);				\
	isync;								\
	msync

#define TLB_UNLOCK							\
	GET_CPUINFO(%r20);						\
	LOAD	%r21, PC_CURTHREAD(%r20);				\
	LOAD	%r22, PC_BOOKE_TLB_LOCK(%r20);				\
									\
	/* update recursion counter */					\
	lwz	%r23, RES_RECURSE(%r22);				\
	subi	%r23, %r23, 1;						\
	stw	%r23, RES_RECURSE(%r22);				\
									\
	cmplwi	%r23, 0;						\
	bne	1f;							\
	isync;								\
	msync;								\
									\
	/* release the lock */						\
	li	%r23, TLB_UNLOCKED;					\
	STORE	%r23, 0(%r22);						\
1:	isync;								\
	msync
#else
#define TLB_LOCK
#define TLB_UNLOCK
#endif	/* SMP */

#define INTERRUPT(label)						\
	.globl	label;							\
	.align	5;							\
	CNAME(label):

/*
 * Interrupt handling routines in BookE can be flexibly placed and do not have
 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
 * all times in order to be able to handle exceptions. We thus arrange for
 * them to be part of kernel text which is always TLB-accessible.
 *
 * The interrupt handling routines have to be 16 bytes aligned: we align them
 * to 32 bytes (cache line length) which supposedly performs better.
 *
 */
	.text
	.globl CNAME(interrupt_vector_base)
	.align 5
interrupt_vector_base:
/*****************************************************************************
 * Catch-all handler to handle uninstalled IVORs
 ****************************************************************************/
INTERRUPT(int_unknown)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
	b	trap_common

/*****************************************************************************
 * Critical input interrupt
 ****************************************************************************/
INTERRUPT(int_critical_input)
	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
	GET_TOCBASE(%r2)
	addi	%r3, %r1, CALLSIZE
	bl	CNAME(powerpc_interrupt)
	TOC_RESTORE
	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
	rfci


/*****************************************************************************
 * Machine check interrupt
 ****************************************************************************/
INTERRUPT(int_machine_check)
	STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
	FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
	GET_TOCBASE(%r2)
	addi	%r3, %r1, CALLSIZE
	bl	CNAME(powerpc_interrupt)
	TOC_RESTORE
	FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
	rfmci


/*****************************************************************************
 * Data storage interrupt
 ****************************************************************************/
INTERRUPT(int_data_storage)
	STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
	b	trap_common


/*****************************************************************************
 * Instruction storage interrupt
 ****************************************************************************/
INTERRUPT(int_instr_storage)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
	b	trap_common


/*****************************************************************************
 * External input interrupt
 ****************************************************************************/
INTERRUPT(int_external_input)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
	b	trap_common


INTERRUPT(int_alignment)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
	b	trap_common


INTERRUPT(int_program)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
	b	trap_common


INTERRUPT(int_fpu)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
	b	trap_common


/*****************************************************************************
 * System call
 ****************************************************************************/
INTERRUPT(int_syscall)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
	b	trap_common


/*****************************************************************************
 * Decrementer interrupt
 ****************************************************************************/
INTERRUPT(int_decrementer)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
	b	trap_common


/*****************************************************************************
 * Fixed interval timer
 ****************************************************************************/
INTERRUPT(int_fixed_interval_timer)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
	b	trap_common


/*****************************************************************************
 * Watchdog interrupt
 ****************************************************************************/
INTERRUPT(int_watchdog)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
	b	trap_common


/*****************************************************************************
 * Altivec Unavailable interrupt
 ****************************************************************************/
INTERRUPT(int_vec)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
	b	trap_common


/*****************************************************************************
 * Altivec Assist interrupt
 ****************************************************************************/
INTERRUPT(int_vecast)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
	b	trap_common


#ifdef __SPE__
/*****************************************************************************
 * Floating point Assist interrupt
 ****************************************************************************/
INTERRUPT(int_spe_fpdata)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPD)
	addi	%r3, %r1, CALLSIZE
	bl	spe_handle_fpdata
	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
	rfi

INTERRUPT(int_spe_fpround)
	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPR)
	addi	%r3, %r1, CALLSIZE
	bl	spe_handle_fpround
	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
	rfi
#endif


#ifdef HWPMC_HOOKS
/*****************************************************************************
 * PMC Interrupt
 ****************************************************************************/
INTERRUPT(int_performance_counter)
	STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
	FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
	b	trap_common
#endif


/*****************************************************************************
 * Data TLB miss interrupt
 *
 * There can be nested TLB misses - while handling a TLB miss we reference
 * data structures that may be not covered by translations. We support up to
 * TLB_NESTED_MAX-1 nested misses.
 * 
 * Registers use:
 *	r31 - dear
 *	r30 - unused
 *	r29 - saved mas0
 *	r28 - saved mas1
 *	r27 - saved mas2
 *	r26 - pmap address
 *	r25 - pte address
 *
 *	r20:r23 - scratch registers
 ****************************************************************************/
INTERRUPT(int_data_tlb_error)
	TLB_PROLOG
	TLB_LOCK

	mfspr	%r31, SPR_DEAR

	/*
	 * Save MAS0-MAS2 registers. There might be another tlb miss during
	 * pte lookup overwriting current contents (which was hw filled).
	 */
	mfspr	%r29, SPR_MAS0
	mfspr	%r28, SPR_MAS1
	mfspr	%r27, SPR_MAS2

	/* Check faulting address. */
	LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS)
	CMPL	cr0, %r31, %r21
	blt	search_user_pmap
	
	/* If it's kernel address, allow only supervisor mode misses. */
	mfsrr1	%r21
	mtcr	%r21
	bt	17, search_failed	/* check MSR[PR] */

#ifdef __powerpc64__
	srdi	%r21, %r31, 48
	cmpldi	cr0, %r21, VM_MIN_KERNEL_ADDRESS@highest
#else
	lis	%r21, VM_MIN_KERNEL_ADDRESS@h
	cmplw	cr0, %r31, %r21
#endif
	blt	search_failed

search_kernel_pmap:
	/* Load r26 with kernel_pmap address */
	bl	1f
#ifdef __powerpc64__
	.llong kernel_pmap_store-.
#else
	.long kernel_pmap_store-.
#endif
1:	mflr	%r21
	LOAD	%r26, 0(%r21)
	add	%r26, %r21, %r26	/* kernel_pmap_store in r26 */

	/* Force kernel tid, set TID to 0 in MAS1. */
	li	%r21, 0
	rlwimi	%r28, %r21, 0, 8, 15	/* clear TID bits */

tlb_miss_handle:
	/* This may result in nested tlb miss. */
	bl	pte_lookup		/* returns PTE address in R25 */

	CMPI	%r25, 0			/* pte found? */
	beq	search_failed

	/* Finish up, write TLB entry. */
	bl	tlb_fill_entry

tlb_miss_return:
	TLB_UNLOCK
	TLB_RESTORE
	rfi

search_user_pmap:
	/* Load r26 with current user space process pmap */
	GET_CPUINFO(%r26)
	LOAD	%r26, PC_CURPMAP(%r26)

	b	tlb_miss_handle

search_failed:
	/*
	 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
	 * the faulting virtual address anyway, but put a fake RPN and no
	 * access rights. This should cause a following {D,I}SI exception.
	 */
	lis	%r23, 0xffff0000@h	/* revoke all permissions */

	/* Load MAS registers. */
	mtspr	SPR_MAS0, %r29
	mtspr	SPR_MAS1, %r28
	mtspr	SPR_MAS2, %r27
	mtspr	SPR_MAS3, %r23

	li	%r23, 0
	mtspr	SPR_MAS7, %r23

	isync
	tlbwe
	msync
	isync
	b	tlb_miss_return

/*****************************************************************************
 *
 * Return pte address that corresponds to given pmap/va.  If there is no valid
 * entry return 0.
 *
 * input: r26 - pmap
 * input: r31 - dear
 * output: r25 - pte address
 *
 * scratch regs used: r21
 *
 ****************************************************************************/
pte_lookup:
	CMPI	%r26, 0
	beq	1f			/* fail quickly if pmap is invalid */

#ifdef __powerpc64__
	rldicl  %r21, %r31, (64 - PP2D_L_L), (64 - PP2D_L_NUM) /* pp2d offset */
	rldicl  %r25, %r31, (64 - PP2D_H_L), (64 - PP2D_H_NUM)
	rldimi  %r21, %r25, PP2D_L_NUM, (64 - (PP2D_L_NUM + PP2D_H_NUM))
	slwi    %r21, %r21, PP2D_ENTRY_SHIFT	/* multiply by pp2d entry size */
	ld	%r25, PM_PP2D(%r26)		/* pmap pm_pp2d[] address */
	ldx	%r25, %r25, %r21		/* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */

	cmpdi   %r25, 0
	beq 2f

	rldicl  %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM)	/* pdir offset */
	slwi    %r21, %r21, PDIR_ENTRY_SHIFT	/* multiply by pdir entry size */
	ldx	%r25, %r25, %r21		/* get ptbl address, i.e.  pmap->pm_pp2d[pp2d_idx][pdir_idx] */

	cmpdi   %r25, 0
	beq     2f

	rldicl  %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */
	slwi    %r21, %r21, PTBL_ENTRY_SHIFT   /* multiply by pte entry size */

#else
	srwi	%r21, %r31, PDIR_SHIFT		/* pdir offset */
	slwi	%r21, %r21, PDIR_ENTRY_SHIFT	/* multiply by pdir entry size */

	lwz	%r25, PM_PDIR(%r26)	/* pmap pm_dir[] address */
	/*
	 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
	 * This load may cause a Data TLB miss for non-kernel pmap!
	 */
	lwzx	%r25, %r25, %r21	/* offset within pm_pdir[] table */
	cmpwi	%r25, 0
	beq	2f

	lis	%r21, PTBL_MASK@h
	ori	%r21, %r21, PTBL_MASK@l
	and	%r21, %r21, %r31

	/* ptbl offset, multiply by ptbl entry size */
	srwi	%r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
#endif

	add	%r25, %r25, %r21		/* address of pte entry */
	/*
	 * Get pte->flags
	 * This load may cause a Data TLB miss for non-kernel pmap!
	 */
	lwz	%r21, PTE_FLAGS(%r25)
	andi.	%r21, %r21, PTE_VALID@l
	bne	2f
1:
	li	%r25, 0
2:
	blr

/*****************************************************************************
 *
 * Load MAS1-MAS3 registers with data, write TLB entry
 *
 * input:
 * r29 - mas0
 * r28 - mas1
 * r27 - mas2
 * r25 - pte
 *
 * output: none
 *
 * scratch regs: r21-r23
 *
 ****************************************************************************/
tlb_fill_entry:
	/*
	 * Update PTE flags: we have to do it atomically, as pmap_protect()
	 * running on other CPUs could attempt to update the flags at the same
	 * time.
	 */
	li	%r23, PTE_FLAGS
1:
	lwarx	%r21, %r23, %r25		/* get pte->flags */
	oris	%r21, %r21, PTE_REFERENCED@h	/* set referenced bit */

	andi.	%r22, %r21, (PTE_SW | PTE_UW)@l	/* check if writable */
	beq	2f
	ori	%r21, %r21, PTE_MODIFIED@l	/* set modified bit */
2:
	stwcx.	%r21, %r23, %r25		/* write it back */
	bne-	1b

	/* Update MAS2. */
	rlwimi	%r27, %r21, 13, 27, 30		/* insert WIMG bits from pte */

	/* Setup MAS3 value in r23. */
	LOAD	%r23, PTE_RPN(%r25)		/* get pte->rpn */
#ifdef __powerpc64__
	rldicr	%r22, %r23, 52, 51		/* extract MAS3 portion of RPN */
	rldicl	%r23, %r23, 20, 54		/* extract MAS7 portion of RPN */

	rlwimi	%r22, %r21, 30, 26, 31		/* insert protection bits from pte */
#else
	rlwinm	%r22, %r23, 20, 0, 11		/* extract MAS3 portion of RPN */

	rlwimi	%r22, %r21, 30, 26, 31		/* insert protection bits from pte */
	rlwimi	%r22, %r21, 20, 12, 19		/* insert lower 8 RPN bits to MAS3 */
	rlwinm	%r23, %r23, 20, 24, 31		/* MAS7 portion of RPN */
#endif

	/* Load MAS registers. */
	mtspr	SPR_MAS0, %r29
	mtspr	SPR_MAS1, %r28
	mtspr	SPR_MAS2, %r27
	mtspr	SPR_MAS3, %r22
	mtspr	SPR_MAS7, %r23

	isync
	tlbwe
	isync
	msync
	blr

/*****************************************************************************
 * Instruction TLB miss interrupt
 *
 * Same notes as for the Data TLB miss
 ****************************************************************************/
INTERRUPT(int_inst_tlb_error)
	TLB_PROLOG
	TLB_LOCK

	mfsrr0	%r31			/* faulting address */

	/*
	 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
	 * lookup overwriting current contents (which was hw filled).
	 */
	mfspr	%r29, SPR_MAS0
	mfspr	%r28, SPR_MAS1
	mfspr	%r27, SPR_MAS2

	mfsrr1	%r21
	mtcr	%r21

	/* check MSR[PR] */
	bt	17, search_user_pmap
	b	search_kernel_pmap


	.globl	interrupt_vector_top
interrupt_vector_top:

/*****************************************************************************
 * Debug interrupt
 ****************************************************************************/
INTERRUPT(int_debug)
	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
	bl	int_debug_int
	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
	rfci

INTERRUPT(int_debug_ed)
	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1)
	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
	bl	int_debug_int
	FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1)
	rfdi
	/* .long 0x4c00004e */

/* Internal helper for debug interrupt handling. */
/* Common code between e500v1/v2 and e500mc-based cores. */
int_debug_int:
	mflr	%r14
	GET_CPUINFO(%r3)
	LOAD	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
	bl	0f
	ADDR(interrupt_vector_base-.)
	ADDR(interrupt_vector_top-.)
0:	mflr	%r5
	LOAD	%r4,0(%r5)	/* interrupt_vector_base in r4 */
	add	%r4,%r4,%r5
	CMPL	cr0, %r3, %r4
	blt	trap_common
	LOAD	%r4,WORD_SIZE(%r5)	/* interrupt_vector_top in r4 */
	add	%r4,%r4,%r5
	addi	%r4,%r4,4
	CMPL	cr0, %r3, %r4
	bge	trap_common
	/* Disable single-stepping for the interrupt handlers. */
	LOAD	%r3, FRAME_SRR1+CALLSIZE(%r1);
	rlwinm	%r3, %r3, 0, 23, 21
	STORE	%r3, FRAME_SRR1+CALLSIZE(%r1);
	/* Restore srr0 and srr1 as they could have been clobbered. */
	GET_CPUINFO(%r4)
	LOAD	%r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4);
	mtspr	SPR_SRR0, %r3
	LOAD	%r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4);
	mtspr	SPR_SRR1, %r4
	mtlr	%r14
	blr

/*****************************************************************************
 * Common trap code
 ****************************************************************************/
trap_common:
	/* Call C trap dispatcher */
	GET_TOCBASE(%r2)
	addi	%r3, %r1, CALLSIZE
	bl	CNAME(powerpc_interrupt)
	TOC_RESTORE

	.globl	CNAME(trapexit)		/* exported for db_backtrace use */
CNAME(trapexit):
	/* disable interrupts */
	wrteei	0

	/* Test AST pending - makes sense for user process only */
	LOAD	%r5, FRAME_SRR1+CALLSIZE(%r1)
	mtcr	%r5
	bf	17, 1f

	GET_CPUINFO(%r3)
	LOAD	%r4, PC_CURTHREAD(%r3)
	lwz	%r4, TD_FLAGS(%r4)
	lis	%r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h
	ori	%r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l
	and.	%r4, %r4, %r5
	beq	1f

	/* re-enable interrupts before calling ast() */
	wrteei	1

	addi	%r3, %r1, CALLSIZE
	bl	CNAME(ast)
	TOC_RESTORE
	.globl	CNAME(asttrapexit)	/* db_backtrace code sentinel #2 */
CNAME(asttrapexit):
	b	trapexit		/* test ast ret value ? */
1:
	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
	rfi


#if defined(KDB)
/*
 * Deliberate entry to dbtrap
 */
	/* .globl	CNAME(breakpoint)*/
ASENTRY_NOPROF(breakpoint)
	mtsprg1	%r1
	mfmsr	%r3
	mtsrr1	%r3
	li	%r4, ~(PSL_EE | PSL_ME)@l
	oris	%r4, %r4, ~(PSL_EE | PSL_ME)@h
	and	%r3, %r3, %r4
	mtmsr	%r3			/* disable interrupts */
	isync
	GET_CPUINFO(%r3)
	STORE	%r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
	STORE	%r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)

	mflr	%r31
	mtsrr0	%r31

	mfspr	%r30, SPR_DEAR
	mfspr	%r31, SPR_ESR
	STORE	%r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
	STORE	%r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)

	mfsrr0	%r30
	mfsrr1	%r31
	STORE	%r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
	STORE	%r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
	isync

	mfcr	%r30

/*
 * Now the kdb trap catching code.
 */
dbtrap:
	FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
/* Call C trap code: */
	GET_TOCBASE(%r2)
	addi	%r3, %r1, CALLSIZE
	bl	CNAME(db_trap_glue)
	TOC_RESTORE
	or.	%r3, %r3, %r3
	bne	dbleave
/* This wasn't for KDB, so switch to real trap: */
	b	trap_common

dbleave:
	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
	rfi
#endif /* KDB */

#ifdef SMP
ENTRY(tlb_lock)
	GET_CPUINFO(%r5)
	LOAD	%r5, PC_CURTHREAD(%r5)
1:	LOADX	%r4, 0, %r3
	CMPI	%r4, TLB_UNLOCKED
	bne	1b
	STOREX	%r5, 0, %r3
	bne-	1b
	isync
	msync
	blr

ENTRY(tlb_unlock)
	isync
	msync
	li	%r4, TLB_UNLOCKED
	STORE	%r4, 0(%r3)
	isync
	msync
	blr

/*
 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
 * only a single word from this granule will actually be used as a spin lock
 * for mutual exclusion between TLB miss handler and pmap layer that
 * manipulates page table contents.
 */
	.data
	.align	5
GLOBAL(tlb0_miss_locks)
	.space	RES_GRANULE * MAXCPU
#endif