diff options
| author | Konstantin Belousov <kib@FreeBSD.org> | 2009-07-09 09:34:11 +0000 |
|---|---|---|
| committer | Konstantin Belousov <kib@FreeBSD.org> | 2009-07-09 09:34:11 +0000 |
| commit | a2622e5dc24f5a23ea124d49b3bd6b1cec647f46 (patch) | |
| tree | 72e1f854630487c503e885bb3cc1df1eeb575521 /sys/amd64/amd64/exception.S | |
| parent | 8e95322a35e75c2ec97a6522b5a70c625a15d6d6 (diff) | |
| download | src-a2622e5dc24f5a23ea124d49b3bd6b1cec647f46.tar.gz src-a2622e5dc24f5a23ea124d49b3bd6b1cec647f46.zip | |
Restore the segment registers and segment base MSRs for amd64 syscall
return path only when neither thread was context switched while
executing syscall code nor syscall explicitely modified LDT or MSRs.
Save segment registers in trap handlers before interrupts are enabled,
to not allow context switches to happen before registers are saved.
Use separated byte in pcb for indication of fast/full return, since
pcb_flags are not synchronized with context switches.
The change puts back syscall microbenchmark numbers that were slowed
down after commit of the support for LDT on amd64.
Reviewed by: jeff
Tested (and tested, and tested ...) by: pho
Approved by: re (kensmith)
Notes
Notes:
svn path=/head/; revision=195486
Diffstat (limited to 'sys/amd64/amd64/exception.S')
| -rw-r--r-- | sys/amd64/amd64/exception.S | 28 |
1 files changed, 19 insertions, 9 deletions
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index daa5c2592988..d78e2341fa36 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -162,19 +162,20 @@ IDTVEC(align) .globl alltraps .type alltraps,@function alltraps: + movq %rdi,TF_RDI(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz alltraps_testi /* already running with kernel GS.base */ swapgs + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) alltraps_testi: testl $PSL_I,TF_RFLAGS(%rsp) - jz alltraps_pushregs + jz alltraps_pushregs_no_rdi sti -alltraps_pushregs: - movq %rdi,TF_RDI(%rsp) alltraps_pushregs_no_rdi: movq %rsi,TF_RSI(%rsp) movq %rdx,TF_RDX(%rsp) @@ -233,14 +234,17 @@ calltrap: .globl alltraps_noen .type alltraps_noen,@function alltraps_noen: + movq %rdi,TF_RDI(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) 1: movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) - jmp alltraps_pushregs + jmp alltraps_pushregs_no_rdi IDTVEC(dblfault) subq $TF_ERR,%rsp @@ -278,12 +282,13 @@ IDTVEC(dblfault) IDTVEC(page) subq $TF_ERR,%rsp movl $T_PAGEFLT,TF_TRAPNO(%rsp) + movq %rdi,TF_RDI(%rsp) /* free up a GP register */ testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs -1: - movq %rdi,TF_RDI(%rsp) /* free up a GP register */ - movq %cr2,%rdi /* preserve %cr2 before .. */ + movq PCPU(CURPCB),%rdi + movb $0,PCB_FULL_IRET(%rdi) +1: movq %cr2,%rdi /* preserve %cr2 before .. */ movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) @@ -311,7 +316,9 @@ IDTVEC(prot) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 2f /* already running with kernel GS.base */ 1: swapgs -2: movw %fs,TF_FS(%rsp) +2: movq PCPU(CURPCB),%rdi + movb $1,PCB_FULL_IRET(%rdi) /* always full iret from GPF */ + movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) @@ -341,6 +348,8 @@ IDTVEC(fast_syscall) movw %gs,TF_GS(%rsp) movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) + movq PCPU(CURPCB),%r11 + movb $0,PCB_FULL_IRET(%r11) sti movq $KUDSEL,TF_SS(%rsp) movq $KUCSEL,TF_CS(%rsp) @@ -644,7 +653,8 @@ doreti_exit: */ testb $SEL_RPL_MASK,TF_CS(%rsp) jz ld_regs - + cmpb $0,PCB_FULL_IRET(%r8) + je ld_regs testl $TF_HASSEGS,TF_FLAGS(%rsp) je set_segs |