diff options
Diffstat (limited to 'sys/i386/include/mptable.h')
| -rw-r--r-- | sys/i386/include/mptable.h | 252 |
1 files changed, 230 insertions, 22 deletions
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index 63fec0e9e9c9..29e9c6eb56fb 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -288,6 +288,14 @@ extern pt_entry_t *SMPpt; struct pcb stoppcbs[MAXCPU]; +#ifdef APIC_IO +/* Variables needed for SMP tlb shootdown. */ +vm_offset_t smp_tlb_addr1; +vm_offset_t smp_tlb_addr2; +volatile int smp_tlb_wait; +static struct mtx smp_tlb_mtx; +#endif + /* * Local data and functions. */ @@ -336,6 +344,9 @@ init_locks(void) #ifdef USE_COMLOCK mtx_init(&com_mtx, "com", NULL, MTX_SPIN); #endif /* USE_COMLOCK */ +#ifdef APIC_IO + mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); +#endif } /* @@ -605,6 +616,10 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + setidt(XINVLPG_OFFSET, Xinvlpg, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + setidt(XINVLRNG_OFFSET, Xinvlrng, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for forwarding hardclock() */ setidt(XHARDCLOCK_OFFSET, Xhardclock, @@ -2190,48 +2205,237 @@ start_ap(int logical_cpu, u_int boot_addr) return 0; /* return FAILURE */ } -#if defined(APIC_IO) && defined(COUNT_XINVLTLB_HITS) -u_int xhits[MAXCPU]; -SYSCTL_OPAQUE(_debug, OID_AUTO, xhits, CTLFLAG_RW, &xhits, sizeof(xhits), - "IU", ""); +#if defined(APIC_IO) + +#ifdef COUNT_XINVLTLB_HITS +u_int xhits_gbl[MAXCPU]; +u_int xhits_pg[MAXCPU]; +u_int xhits_rng[MAXCPU]; +SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, + sizeof(xhits_gbl), "IU", ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, + sizeof(xhits_pg), "IU", ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, + sizeof(xhits_rng), "IU", ""); + +u_int ipi_global; +u_int ipi_page; +u_int ipi_range; +u_int ipi_range_size; +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, + 0, ""); + +u_int ipi_masked_global; +u_int ipi_masked_page; +u_int ipi_masked_range; +u_int ipi_masked_range_size; +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, + &ipi_masked_global, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, + &ipi_masked_page, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, + &ipi_masked_range, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, + &ipi_masked_range_size, 0, ""); #endif /* * Flush the TLB on all other CPU's + */ +static void +smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) +{ + u_int ncpu; + register_t eflags; + + ncpu = mp_ncpus - 1; /* does not shootdown self */ + if (ncpu < 1) + return; /* no other cpus */ + eflags = read_eflags(); + if ((eflags & PSL_I) == 0) + panic("absolutely cannot call smp_ipi_shootdown with interrupts already disabled"); + mtx_lock_spin(&smp_tlb_mtx); + smp_tlb_addr1 = addr1; + smp_tlb_addr2 = addr2; + atomic_store_rel_int(&smp_tlb_wait, 0); + ipi_all_but_self(vector); + while (smp_tlb_wait < ncpu) + ia32_pause(); + mtx_unlock_spin(&smp_tlb_mtx); +} + +/* + * This is about as magic as it gets. fortune(1) has got similar code + * for reversing bits in a word. Who thinks up this stuff?? + * + * Yes, it does appear to be consistently faster than: + * while (i = ffs(m)) { + * m >>= i; + * bits++; + * } + * and + * while (lsb = (m & -m)) { // This is magic too + * m &= ~lsb; // or: m ^= lsb + * bits++; + * } + * Both of these latter forms do some very strange things on gcc-3.1 with + * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2. + * There is probably an SSE or MMX popcnt instruction. * - * XXX: Needs to handshake and wait for completion before proceding. + * I wonder if this should be in libkern? + * + * XXX Stop the presses! Another one: + * static __inline u_int32_t + * popcnt1(u_int32_t v) + * { + * v -= ((v >> 1) & 0x55555555); + * v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + * v = (v + (v >> 4)) & 0x0F0F0F0F; + * return (v * 0x01010101) >> 24; + * } + * The downside is that it has a multiply. With a pentium3 with + * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use + * an imull, and in that case it is faster. In most other cases + * it appears slightly slower. */ +static __inline u_int32_t +popcnt(u_int32_t m) +{ + + m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1); + m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2); + m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4); + m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8); + m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16); + return m; +} + +static void +smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) +{ + int ncpu, othercpus; + register_t eflags; + + othercpus = mp_ncpus - 1; + if (mask == (u_int)-1) { + ncpu = othercpus; + if (ncpu < 1) + return; + } else { + /* XXX there should be a pcpu self mask */ + mask &= ~(1 << PCPU_GET(cpuid)); + if (mask == 0) + return; + ncpu = popcnt(mask); + if (ncpu > othercpus) { + /* XXX this should be a panic offence */ + printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", + ncpu, othercpus); + ncpu = othercpus; + } + /* XXX should be a panic, implied by mask == 0 above */ + if (ncpu < 1) + return; + } + eflags = read_eflags(); + if ((eflags & PSL_I) == 0) + panic("absolutely cannot call smp_targeted_ipi_shootdown with interrupts already disabled"); + mtx_lock_spin(&smp_tlb_mtx); + smp_tlb_addr1 = addr1; + smp_tlb_addr2 = addr2; + atomic_store_rel_int(&smp_tlb_wait, 0); + if (mask == (u_int)-1) + ipi_all_but_self(vector); + else + ipi_selected(mask, vector); + while (smp_tlb_wait < ncpu) + ia32_pause(); + mtx_unlock_spin(&smp_tlb_mtx); +} +#endif + void smp_invltlb(void) { #if defined(APIC_IO) - if (smp_started) - ipi_all_but_self(IPI_INVLTLB); + if (smp_started) { + smp_tlb_shootdown(IPI_INVLTLB, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_global++; +#endif + } #endif /* APIC_IO */ } void -invlpg(u_int addr) +smp_invlpg(vm_offset_t addr) { - __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); +#if defined(APIC_IO) + if (smp_started) { + smp_tlb_shootdown(IPI_INVLPG, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_page++; +#endif + } +#endif /* APIC_IO */ +} - /* send a message to the other CPUs */ - smp_invltlb(); +void +smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) +{ +#if defined(APIC_IO) + if (smp_started) { + smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_range++; + ipi_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif + } +#endif /* APIC_IO */ } void -invltlb(void) +smp_masked_invltlb(u_int mask) { - u_long temp; +#if defined(APIC_IO) + if (smp_started) { + smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_global++; +#endif + } +#endif /* APIC_IO */ +} - /* - * This should be implemented as load_cr3(rcr3()) when load_cr3() is - * inlined. - */ - __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); +void +smp_masked_invlpg(u_int mask, vm_offset_t addr) +{ +#if defined(APIC_IO) + if (smp_started) { + smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_page++; +#endif + } +#endif /* APIC_IO */ +} - /* send a message to the other CPUs */ - smp_invltlb(); +void +smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) +{ +#if defined(APIC_IO) + if (smp_started) { + smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_range++; + ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif + } +#endif /* APIC_IO */ } @@ -2251,7 +2455,7 @@ ap_init(void) /* spin */ ; /* BSP may have changed PTD while we were waiting */ - cpu_invltlb(); + invltlb(); #if defined(I586_CPU) && !defined(NO_F00F_HACK) lidt(&r_idt); @@ -2290,6 +2494,9 @@ ap_init(void) /* Build our map of 'other' CPUs. */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); + if (bootverbose) + apic_dump("ap_init()"); + printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); if (smp_cpus == mp_ncpus) { @@ -2325,7 +2532,8 @@ forwarded_statclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); + statclock_process(curthread->td_kse, TRAPF_PC(&frame), + TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } |