diff options
| author | Konstantin Belousov <kib@FreeBSD.org> | 2022-09-08 20:15:32 +0000 |
|---|---|---|
| committer | Konstantin Belousov <kib@FreeBSD.org> | 2022-09-14 15:46:32 +0000 |
| commit | 49587b8fb4d12166d30efb7111d32f4606a1e73b (patch) | |
| tree | 8af18ec311f7814683e6333a1b1637dd858a9a33 | |
| parent | fd25c62278cea3492a14abadc190f43da5f74224 (diff) | |
i386 copyout/in_fast: handle page fault from KVA access
by delegating the work to the slow path.
Some kernel memory, like pipe buffers, is pageable. We must not enable
interrupts, and consequently, preemption, while in critical section in
the fast copyout path, because we use pcpu buffers. If page fault
occurs while copying from the pcpu copyout_buf to kernel memory, abort
fast path and delegate work to the slow implementation.
In collaboration with: pho, tijl
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
| -rw-r--r-- | sys/i386/i386/copyout_fast.s | 21 | ||||
| -rw-r--r-- | sys/i386/i386/exception.s | 25 |
2 files changed, 24 insertions, 22 deletions
diff --git a/sys/i386/i386/copyout_fast.s b/sys/i386/i386/copyout_fast.s index 70647fe7613d..4391f36b18e4 100644 --- a/sys/i386/i386/copyout_fast.s +++ b/sys/i386/i386/copyout_fast.s @@ -47,32 +47,30 @@ ENTRY(copyout_fast) pushl %ebx movl 20(%ebp),%ebx /* KCR3 */ - movl PCPU(CURPCB),%edx - movl PCB_CR3(%edx),%edx /* UCR3 */ /* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */ movl 16(%ebp),%ecx movl 8(%ebp),%esi + movl %esp,%eax + movl $copyout_fault,%edx + cli movl PCPU(COPYOUT_BUF),%edi - rep; movsb +pf_y1: rep; movsb movl 16(%ebp),%ecx /* len */ movl PCPU(COPYOUT_BUF),%esi /* kaddr */ movl 12(%ebp),%edi /* uaddr */ - - movl %esp,%eax movl PCPU(TRAMPSTK),%esp - + movl PCPU(CURPCB),%edx + movl PCB_CR3(%edx),%edx /* UCR3 */ movl %edx,%cr3 movl $copyout_fault,%edx - /* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */ pf_x1: rep; movsb movl %ebx,%cr3 movl %eax,%esp sti - xorl %eax,%eax popl %ebx popl %edi @@ -93,10 +91,10 @@ ENTRY(copyin_fast) movl PCB_CR3(%eax),%edx /* UCR3 */ movl 16(%ebp),%ecx /* len */ movl 8(%ebp),%esi /* udaddr */ + movl %esp,%eax + cli movl PCPU(COPYOUT_BUF),%edi /* kaddr */ - - movl %esp,%eax movl PCPU(TRAMPSTK),%esp movl %edx,%cr3 movl $copyout_fault,%edx @@ -110,10 +108,9 @@ pf_x2: rep; movsb movl 16(%ebp),%ecx movl 12(%ebp),%edi movl PCPU(COPYOUT_BUF),%esi - rep; movsb +pf_y2: rep; movsb sti - xorl %eax,%eax popl %ebx popl %edi diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index f8d61c884a21..e83d95e6afe1 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -130,15 +130,15 @@ IDTVEC(prot) jmp irettraps IDTVEC(page) testl $PSL_VM, TF_EFLAGS-TF_ERR(%esp) - jnz 4f + jnz 5f testb $SEL_RPL_MASK, TF_CS-TF_ERR(%esp) - jnz 4f + jnz 5f cmpl $PMAP_TRM_MIN_ADDRESS, TF_EIP-TF_ERR(%esp) - jb 4f + jb 5f pushl %eax movl TF_EIP-TF_ERR+4(%esp), %eax addl $1f, %eax - call 5f + call 6f 1: cmpl $pf_x1, %eax je 2f cmpl $pf_x2, %eax @@ -154,16 +154,21 @@ IDTVEC(page) cmpl $pf_x7, %eax je 2f cmpl $pf_x8, %eax - jne 3f -2: popl %eax - movl %ebx, %cr3 + je 2f + cmpl $pf_y1, %eax + je 3f + cmpl $pf_y2, %eax + je 3f + jmp 4f +2: movl %ebx, %cr3 +3: popl %eax movl %edx, TF_EIP-TF_ERR(%esp) addl $4, %esp iret -3: popl %eax -4: pushl $T_PAGEFLT +4: popl %eax +5: pushl $T_PAGEFLT jmp alltraps -5: subl (%esp), %eax +6: subl (%esp), %eax retl IDTVEC(rsvd_pti) IDTVEC(rsvd) |