/*-
* Copyright (c) 1989, 1990 William F. Jolitz.
* Copyright (c) 1990 The Regents of the University of California.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from: vector.s, 386BSD 0.1 unknown origin
* $FreeBSD$
*/
/*
* Interrupt entry points for external interrupts triggered by I/O APICs
* as well as IPI handlers.
*/
#include "opt_smp.h"
#include <machine/asmacros.h>
#include <machine/apicreg.h>
#include "assym.s"
/*
* I/O Interrupt Entry Point. Rather than having one entry point for
* each interrupt source, we use one entry point for each 32-bit word
* in the ISR. The handler determines the highest bit set in the ISR,
* translates that into a vector, and passes the vector to the
* lapic_handle_intr() function.
*/
#define ISR_VEC(index, vec_name) \
.text ; \
SUPERALIGN_TEXT ; \
IDTVEC(vec_name) ; \
PUSH_FRAME ; \
FAKE_MCOUNT(TF_RIP(%rsp)) ; \
movq lapic, %rdx ; /* pointer to local APIC */ \
movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
jz 2f ; \
addl $(32 * index),%eax ; \
1: ; \
movq %rsp, %rsi ; \
movl %eax, %edi ; /* pass the IRQ */ \
call lapic_handle_intr ; \
MEXITCOUNT ; \
jmp doreti ; \
2: movl $-1, %eax ; /* send a vector of -1 */ \
jmp 1b
/*
* Handle "spurious INTerrupts".
* Notes:
* This is different than the "spurious INTerrupt" generated by an
* 8259 PIC for missing INTs. See the APIC documentation for details.
* This routine should NOT do an 'EOI' cycle.
*/
.text
SUPERALIGN_TEXT
IDTVEC(spuriousint)
/* No EOI cycle used here */
jmp doreti_iret
ISR_VEC(1, apic_isr1)
ISR_VEC(2, apic_isr2)
ISR_VEC(3, apic_isr3)
ISR_VEC(4, apic_isr4)
ISR_VEC(5, apic_isr5)
ISR_VEC(6, apic_isr6)
ISR_VEC(7, apic_isr7)
/*
* Local APIC periodic timer handler.
*/
.text
SUPERALIGN_TEXT
IDTVEC(timerint)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp, %rdi
call lapic_handle_timer
MEXITCOUNT
jmp doreti
/*
* Local APIC CMCI handler.
*/
.text
SUPERALIGN_TEXT
IDTVEC(cmcint)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))
call lapic_handle_cmc
MEXITCOUNT
jmp doreti
/*
* Local APIC error interrupt handler.
*/
.text
SUPERALIGN_TEXT
IDTVEC(errorint)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))
call lapic_handle_error
MEXITCOUNT
jmp doreti
#ifdef SMP
/*
* Global address space TLB shootdown.
*/
.text
SUPERALIGN_TEXT
IDTVEC(invltlb)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_gbl(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invltlb_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
movq %cr3, %rax /* invalidate the TLB */
movq %rax, %cr3
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rax
jmp doreti_iret
/*
* Single page TLB shootdown
*/
.text
SUPERALIGN_TEXT
IDTVEC(invlpg)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_pg(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invlpg_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
movq smp_tlb_addr1, %rax
invlpg (%rax) /* invalidate single page */
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rax
jmp doreti_iret
/*
* Page range TLB shootdown.
*/
.text
SUPERALIGN_TEXT
IDTVEC(invlrng)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_rng(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invlrng_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
pushq %rdx
movq smp_tlb_addr1, %rdx
movq smp_tlb_addr2, %rax
1: invlpg (%rdx) /* invalidate single page */
addq $PAGE_SIZE, %rdx
cmpq %rax, %rdx
jb 1b
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rdx
popq %rax
jmp doreti_iret
/*
* Invalidate cache.
*/
.text
SUPERALIGN_TEXT
IDTVEC(invlcache)
#ifdef COUNT_IPIS
PUSH_FRAME
movl PCPU(CPUID), %eax
movq ipi_invlcache_counts(,%rax,8),%rax
incq (%rax)
POP_FRAME
#endif
pushq %rax
wbinvd
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rax
jmp doreti_iret
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.
*/
.text
SUPERALIGN_TEXT
IDTVEC(ipi_intr_bitmap_handler)
PUSH_FRAME
movq lapic, %rdx
movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
FAKE_MCOUNT(TF_RIP(%rsp))
call ipi_bitmap_handler
MEXITCOUNT
jmp doreti
/*
* Executed by a CPU when it receives an IPI_STOP from another CPU.
*/
.text
SUPERALIGN_TEXT
IDTVEC(cpustop)
PUSH_FRAME
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
call cpustop_handler
jmp doreti
/*
* Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
*/
.text
SUPERALIGN_TEXT
IDTVEC(cpususpend)
PUSH_FRAME
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
call cpususpend_handler
POP_FRAME
jmp doreti_iret
/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
*
* - Calls the generic rendezvous action function.
*/
.text
SUPERALIGN_TEXT
IDTVEC(rendezvous)
PUSH_FRAME
#ifdef COUNT_IPIS
movl PCPU(CPUID), %eax
movq ipi_rendezvous_counts(,%rax,8), %rax
incq (%rax)
#endif
call smp_rendezvous_action
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
jmp doreti
#endif /* SMP */
