diff options
| -rw-r--r-- | sys/conf/files.riscv | 1 | ||||
| -rw-r--r-- | sys/riscv/include/elf.h | 1 | ||||
| -rw-r--r-- | sys/riscv/include/md_var.h | 1 | ||||
| -rw-r--r-- | sys/riscv/include/pcb.h | 10 | ||||
| -rw-r--r-- | sys/riscv/include/riscvreg.h | 8 | ||||
| -rw-r--r-- | sys/riscv/include/ucontext.h | 22 | ||||
| -rw-r--r-- | sys/riscv/include/vector.h | 36 | ||||
| -rw-r--r-- | sys/riscv/riscv/exec_machdep.c | 187 | ||||
| -rw-r--r-- | sys/riscv/riscv/identcpu.c | 11 | ||||
| -rw-r--r-- | sys/riscv/riscv/machdep.c | 3 | ||||
| -rw-r--r-- | sys/riscv/riscv/swtch.S | 55 | ||||
| -rw-r--r-- | sys/riscv/riscv/trap.c | 15 | ||||
| -rw-r--r-- | sys/riscv/riscv/vector.c | 240 | ||||
| -rw-r--r-- | sys/riscv/riscv/vm_machdep.c | 14 |
14 files changed, 587 insertions, 17 deletions
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 58a31b5f326e..307276b5fd52 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -97,6 +97,7 @@ riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack +riscv/riscv/vector.c standard riscv/riscv/vm_machdep.c standard riscv/vmm/vmm.c optional vmm riscv/vmm/vmm_aplic.c optional vmm diff --git a/sys/riscv/include/elf.h b/sys/riscv/include/elf.h index 78788abe1e57..aa5c1318f088 100644 --- a/sys/riscv/include/elf.h +++ b/sys/riscv/include/elf.h @@ -81,6 +81,7 @@ __ElfType(Auxinfo); #define HWCAP_ISA_D HWCAP_ISA_BIT('d') #define HWCAP_ISA_C HWCAP_ISA_BIT('c') #define HWCAP_ISA_H HWCAP_ISA_BIT('h') +#define HWCAP_ISA_V HWCAP_ISA_BIT('v') #define HWCAP_ISA_G \ (HWCAP_ISA_I | HWCAP_ISA_M | HWCAP_ISA_A | HWCAP_ISA_F | HWCAP_ISA_D) #define HWCAP_ISA_B HWCAP_ISA_BIT('b') diff --git a/sys/riscv/include/md_var.h b/sys/riscv/include/md_var.h index 85a51e30b4a7..5f921ed500bf 100644 --- a/sys/riscv/include/md_var.h +++ b/sys/riscv/include/md_var.h @@ -46,6 +46,7 @@ extern bool has_hyp; extern bool has_sstc; extern bool has_sscofpmf; extern bool has_svpbmt; +extern bool has_vector; struct dumperinfo; struct minidumpstate; diff --git a/sys/riscv/include/pcb.h b/sys/riscv/include/pcb.h index bb88516a1385..c0d70d9ab107 100644 --- a/sys/riscv/include/pcb.h +++ b/sys/riscv/include/pcb.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -51,6 +51,14 @@ struct pcb { #define PCB_FP_STARTED 0x1 #define PCB_FP_USERMASK 0x1 vm_offset_t pcb_onfault; /* Copyinout fault handler */ + /* Vector state. */ + uint64_t pcb_vsflags; +#define PCB_VS_STARTED 0x1 + uint64_t pcb_vstart; + uint64_t pcb_vl; + uint64_t pcb_vtype; + uint64_t pcb_vcsr; + void *pcb_vsaved; /* Vector registers area. */ }; #ifdef _KERNEL diff --git a/sys/riscv/include/riscvreg.h b/sys/riscv/include/riscvreg.h index 23feb419d04c..8a81e5e6c2e4 100644 --- a/sys/riscv/include/riscvreg.h +++ b/sys/riscv/include/riscvreg.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2024 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -64,6 +64,12 @@ #define SSTATUS_SPIE_SHIFT 5 #define SSTATUS_SPP (1 << 8) #define SSTATUS_SPP_SHIFT 8 +#define SSTATUS_VS_SHIFT 9 +#define SSTATUS_VS_OFF (0x0 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_INITIAL (0x1 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_CLEAN (0x2 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_DIRTY (0x3 << SSTATUS_VS_SHIFT) +#define SSTATUS_VS_MASK (0x3 << SSTATUS_VS_SHIFT) #define SSTATUS_FS_SHIFT 13 #define SSTATUS_FS_OFF (0x0 << SSTATUS_FS_SHIFT) #define SSTATUS_FS_INITIAL (0x1 << SSTATUS_FS_SHIFT) diff --git a/sys/riscv/include/ucontext.h b/sys/riscv/include/ucontext.h index 13242c457afe..cf6a0bc1ff16 100644 --- a/sys/riscv/include/ucontext.h +++ b/sys/riscv/include/ucontext.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -54,13 +54,31 @@ struct fpregs { int pad; }; +struct riscv_reg_context { + __uint32_t ctx_id; + __uint32_t ctx_size; +}; + +#define RISCV_CTX_MAGIC_END 0 +#define RISCV_CTX_MAGIC_VS 0x53465457 + +struct vector_context { + struct riscv_reg_context ctx; + uint64_t vs_vstart; + uint64_t vs_vl; + uint64_t vs_vtype; + uint64_t vs_vcsr; + uint64_t reserved[3]; +}; + struct __mcontext { struct gpregs mc_gpregs; struct fpregs mc_fpregs; int mc_flags; #define _MC_FP_VALID 0x1 /* Set when mc_fpregs has valid data */ int mc_pad; - __uint64_t mc_spare[8]; /* Space for expansion */ + __uint64_t mc_ptr; /* Address of ctx headers and data */ + __uint64_t mc_spare[7]; /* Space for expansion */ }; typedef struct __mcontext mcontext_t; diff --git a/sys/riscv/include/vector.h b/sys/riscv/include/vector.h new file mode 100644 index 000000000000..a998598c61b6 --- /dev/null +++ b/sys/riscv/include/vector.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_VECTOR_H_ +#define _MACHINE_VECTOR_H_ + +void vector_state_init(struct thread *td); +void vector_state_store(struct thread *td); +void vector_state_store_savectx(struct pcb *p); +void vector_state_restore(struct thread *td); +void vector_copy_thread(struct thread *td1, struct thread *td2); +int vector_get_size(void); + +#endif /* !_MACHINE_VECTOR_H_ */ diff --git a/sys/riscv/riscv/exec_machdep.c b/sys/riscv/riscv/exec_machdep.c index 4e0d1f2a7e5c..bf5488249d5c 100644 --- a/sys/riscv/riscv/exec_machdep.c +++ b/sys/riscv/riscv/exec_machdep.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2014 Andrew Turner - * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -62,6 +62,8 @@ #include <machine/riscvreg.h> #include <machine/sbi.h> #include <machine/trap.h> +#include <machine/vector.h> +#include <machine/md_var.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -71,6 +73,10 @@ static void get_fpcontext(struct thread *td, mcontext_t *mcp); static void set_fpcontext(struct thread *td, mcontext_t *mcp); +#define CTX_SIZE_VS(buf_size) \ + roundup2(sizeof(struct vector_context) + (buf_size), \ + _Alignof(struct vector_context)) + _Static_assert(sizeof(mcontext_t) == 864, "mcontext_t size incorrect"); _Static_assert(sizeof(ucontext_t) == 936, "ucontext_t size incorrect"); _Static_assert(sizeof(siginfo_t) == 80, "siginfo_t size incorrect"); @@ -227,10 +233,55 @@ get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) return (0); } +static int +restore_vector_state(struct pcb *pcb, struct riscv_reg_context *ctx, + vm_offset_t addr) +{ + struct vector_context vs_ctx; + size_t buf_size; + int error; + + /* Ensure vector engine present. */ + if (!has_vector) + return (EINVAL); + + /* Ensure vector state is initialized. */ + if (pcb->pcb_vsaved == NULL) + return (EINVAL); + + buf_size = vector_get_size(); + if (ctx->ctx_size != CTX_SIZE_VS(buf_size)) + return (EINVAL); + + /* Copy the vector registers. */ + error = copyin((const void *)addr, &vs_ctx, sizeof(vs_ctx)); + if (error != 0) + return (error); + + /* Copy the vector data. */ + if (copyin((void *)(addr + sizeof(vs_ctx)), pcb->pcb_vsaved, + buf_size) != 0) + return (EINVAL); + + /* Restore pcb registers. */ + pcb->pcb_vstart = vs_ctx.vs_vstart; + pcb->pcb_vl = vs_ctx.vs_vl; + pcb->pcb_vtype = vs_ctx.vs_vtype; + pcb->pcb_vcsr = vs_ctx.vs_vcsr; + pcb->pcb_vsflags |= PCB_VS_STARTED; + + return (0); +} + int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf; + struct riscv_reg_context ctx; + struct pcb *pcb; + vm_offset_t addr; + int error, seen_types; + bool done; tf = td->td_frame; @@ -256,8 +307,48 @@ set_mcontext(struct thread *td, mcontext_t *mcp) tf->tf_gp = mcp->mc_gpregs.gp_gp; tf->tf_sepc = mcp->mc_gpregs.gp_sepc; tf->tf_sstatus = mcp->mc_gpregs.gp_sstatus; + set_fpcontext(td, mcp); + if (mcp->mc_ptr == 0) + return (0); + + /* Read any register contexts we find */ + addr = mcp->mc_ptr; + pcb = td->td_pcb; + +#define CTX_TYPE_VS (1 << 0) + + seen_types = 0; + done = false; + do { + if (!__is_aligned(addr, _Alignof(struct riscv_reg_context))) + return (EINVAL); + + error = copyin((const void *)addr, &ctx, sizeof(ctx)); + if (error != 0) + return (error); + + switch (ctx.ctx_id) { + case RISCV_CTX_MAGIC_VS: + if ((seen_types & CTX_TYPE_VS) != 0) + return (EINVAL); + seen_types |= CTX_TYPE_VS; + error = restore_vector_state(pcb, &ctx, addr); + if (error) + return (EINVAL); + break; + case RISCV_CTX_MAGIC_END: + done = true; + break; + default: + return (EINVAL); + } + addr += ctx.ctx_size; + } while (!done); + +#undef CTX_TYPE_VS + return (0); } @@ -333,6 +424,71 @@ sys_sigreturn(struct thread *td, struct sigreturn_args *uap) return (EJUSTRETURN); } +static bool +sendsig_ctx_end(struct thread *td, vm_offset_t *addrp) +{ + struct riscv_reg_context end_ctx; + vm_offset_t ctx_addr; + + *addrp -= sizeof(end_ctx); + ctx_addr = *addrp; + + memset(&end_ctx, 0, sizeof(end_ctx)); + end_ctx.ctx_id = RISCV_CTX_MAGIC_END; + end_ctx.ctx_size = sizeof(end_ctx); + + if (copyout(&end_ctx, (void *)ctx_addr, sizeof(end_ctx)) != 0) + return (false); + return (true); +} + +static bool +sendsig_ctx_vector(struct thread *td, vm_offset_t *addrp) +{ + struct vector_context vs_ctx; + struct pcb *pcb; + size_t buf_size, ctx_size; + vm_offset_t vs_ctx_addr; + + pcb = td->td_pcb; + /* Do nothing if vector hasn't started */ + if (pcb->pcb_vsaved == NULL) + return (true); + + MPASS(pcb->pcb_vsaved != NULL); + + buf_size = vector_get_size(); + ctx_size = CTX_SIZE_VS(buf_size); + + /* Address for the full context. */ + *addrp -= ctx_size; + vs_ctx_addr = *addrp; + + memset(&vs_ctx, 0, sizeof(vs_ctx)); + vs_ctx.ctx.ctx_id = RISCV_CTX_MAGIC_VS; + vs_ctx.ctx.ctx_size = ctx_size; + vs_ctx.vs_vstart = pcb->pcb_vstart; + vs_ctx.vs_vl = pcb->pcb_vl; + vs_ctx.vs_vtype = pcb->pcb_vtype; + vs_ctx.vs_vcsr = pcb->pcb_vcsr; + + /* Copy out the header and data */ + if (copyout(&vs_ctx, (void *)vs_ctx_addr, sizeof(vs_ctx)) != 0) + return (false); + if (copyout(pcb->pcb_vsaved, (void *)(vs_ctx_addr + sizeof(vs_ctx)), + buf_size) != 0) + return (false); + + return (true); +} + +typedef bool(*ctx_func)(struct thread *, vm_offset_t *); +static const ctx_func ctx_funcs[] = { + sendsig_ctx_end, /* Must go first. */ + sendsig_ctx_vector, + NULL, +}; + void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { @@ -342,8 +498,10 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) struct sigacts *psp; struct thread *td; struct proc *p; + vm_offset_t addr; int onstack; int sig; + int i; td = curthread; p = td->td_proc; @@ -362,16 +520,12 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { - fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + + addr = ((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); } else { - fp = (struct sigframe *)td->td_frame->tf_sp; + addr = td->td_frame->tf_sp; } - /* Make room, keeping the stack aligned */ - fp--; - fp = STACKALIGN(fp); - /* Fill in the frame to copy out */ bzero(&frame, sizeof(frame)); get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); @@ -383,6 +537,25 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); + for (i = 0; ctx_funcs[i] != NULL; i++) { + if (!ctx_funcs[i](td, &addr)) { + CTR4(KTR_SIG, + "sendsig: frame sigexit td=%p fp=%#lx func[%d]=%p", + td, addr, i, ctx_funcs[i]); + PROC_LOCK(p); + sigexit(td, SIGILL); + /* NOTREACHED */ + } + } + + /* Point at the first context */ + frame.sf_uc.uc_mcontext.mc_ptr = addr; + + /* Make room, keeping the stack aligned */ + fp = (struct sigframe *)addr; + fp--; + fp = (struct sigframe *)STACKALIGN(fp); + /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ diff --git a/sys/riscv/riscv/identcpu.c b/sys/riscv/riscv/identcpu.c index af71cc2f89f0..7b2d9a1d1259 100644 --- a/sys/riscv/riscv/identcpu.c +++ b/sys/riscv/riscv/identcpu.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * - * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * Copyright (c) 2022 Mitchell Horne <mhorne@FreeBSD.org> * Copyright (c) 2023 The FreeBSD Foundation @@ -75,6 +75,7 @@ u_int mmu_caps; /* Supervisor-mode extension support. */ bool has_hyp; +bool has_vector; bool __read_frequently has_sstc; bool __read_frequently has_sscofpmf; bool has_svpbmt; @@ -280,6 +281,7 @@ parse_riscv_isa(struct cpu_desc *desc, char *isa, int len) case 'h': case 'i': case 'm': + case 'v': desc->isa_extensions |= HWCAP_ISA_BIT(isa[i]); i++; break; @@ -402,6 +404,9 @@ identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc) return; } + if (bootverbose) + printf("hart %d isa: %s\n", hart, isa); + /* * The string is specified to be lowercase, but let's be * certain. @@ -462,6 +467,7 @@ update_global_capabilities(u_int cpu, struct cpu_desc *desc) UPDATE_CAP(mmu_caps, desc->mmu_caps); /* Supervisor-mode extension support. */ + UPDATE_CAP(has_vector, (desc->isa_extensions & HWCAP_ISA_V) != 0); UPDATE_CAP(has_hyp, (desc->isa_extensions & HWCAP_ISA_H) != 0); UPDATE_CAP(has_sstc, (desc->smode_extensions & SV_SSTC) != 0); UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0); @@ -605,7 +611,8 @@ printcpuinfo(u_int cpu) "\04Double" "\06Float" "\10Hypervisor" - "\15Mult/Div"); + "\15Mult/Div" + "\26Vector"); } if (SHOULD_PRINT(smode_extensions)) { diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c index f5479c479109..99cede3efba6 100644 --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2014 Andrew Turner - * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -299,6 +299,7 @@ init_proc0(void *kstack) thread0.td_kstack_pages = KSTACK_PAGES; thread0.td_pcb = &pcb0; thread0.td_pcb->pcb_fpflags = 0; + thread0.td_pcb->pcb_vsflags = 0; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; } diff --git a/sys/riscv/riscv/swtch.S b/sys/riscv/riscv/swtch.S index fc9b493744b8..b7b2e1ec9dd3 100644 --- a/sys/riscv/riscv/swtch.S +++ b/sys/riscv/riscv/swtch.S @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -210,6 +210,17 @@ ENTRY(cpu_throw) call _C_LABEL(pmap_activate_sw) mv a0, s0 + /* Is VS enabled for new thread? */ + ld t0, TD_FRAME(a0) + ld t1, (TF_SSTATUS)(t0) + li t2, SSTATUS_VS_MASK + and t3, t1, t2 + beqz t3, 1f /* No, skip. */ + + call _C_LABEL(vector_state_restore) + mv a0, s0 + +1: /* Store the new curthread */ sd a0, PC_CURTHREAD(tp) /* And the new pcb */ @@ -304,6 +315,33 @@ ENTRY(cpu_switch) __fpe_state_save x13 1: + /* + * Is VS enabled and is it in dirty state + * for the old thread? + */ + ld t0, TD_FRAME(a0) + ld t1, (TF_SSTATUS)(t0) + li t2, SSTATUS_VS_MASK + and t3, t1, t2 + li t2, SSTATUS_VS_DIRTY + bne t3, t2, 1f /* No, skip. */ + + /* Yes, mark VS clean and save registers. */ + li t2, ~SSTATUS_VS_MASK + and t3, t1, t2 + li t2, SSTATUS_VS_CLEAN + or t3, t3, t2 + sd t3, (TF_SSTATUS)(t0) + + mv s0, a0 + mv s1, a1 + mv s2, a2 + call _C_LABEL(vector_state_store) + mv a0, s0 + mv a1, s1 + mv a2, s2 +1: + /* Activate the new thread's pmap */ mv s0, a0 mv s1, a1 @@ -321,6 +359,20 @@ ENTRY(cpu_switch) ld t0, TD_LOCK(a1) beq t0, s2, 1b #endif + + /* Is VS enabled for new thread? */ + ld t0, TD_FRAME(a1) + ld t1, (TF_SSTATUS)(t0) + li t2, SSTATUS_VS_MASK + and t3, t1, t2 + beqz t3, 1f /* No, skip. */ + + /* Restore vector registers. */ + mv a0, a1 + call _C_LABEL(vector_state_restore) + mv a1, s1 +1: + /* * Restore the saved context. */ @@ -453,5 +505,6 @@ ENTRY(savectx) sd s11, (PCB_S + 11 * 8)(a0) __fpe_state_save a0 + call _C_LABEL(vector_state_store_savectx) ret END(savectx) diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c index d6df1245a24e..4ca3e031670e 100644 --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -62,6 +62,8 @@ #include <machine/frame.h> #include <machine/pcb.h> #include <machine/pcpu.h> +#include <machine/vector.h> +#include <machine/md_var.h> #include <machine/resource.h> @@ -443,6 +445,17 @@ do_trap_user(struct trapframe *frame) pcb->pcb_fpflags |= PCB_FP_STARTED; break; } + if (has_vector && (pcb->pcb_vsflags & PCB_VS_STARTED) == 0) { + /* + * Could be a vector trap. Enable VS usage + * for this thread and try again. + */ + vector_state_init(td); + frame->tf_sstatus &= ~SSTATUS_VS_MASK; + frame->tf_sstatus |= SSTATUS_VS_CLEAN; + pcb->pcb_vsflags |= PCB_VS_STARTED; + break; + } call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc, exception); userret(td, frame); diff --git a/sys/riscv/riscv/vector.c b/sys/riscv/riscv/vector.c new file mode 100644 index 000000000000..9c57cefd5a4b --- /dev/null +++ b/sys/riscv/riscv/vector.c @@ -0,0 +1,240 @@ +/*- + * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/eventhandler.h> +#include <sys/kernel.h> +#include <sys/proc.h> + +#include <vm/uma.h> + +#include <machine/vector.h> +#include <machine/pcb.h> +#include <machine/reg.h> +#include <machine/md_var.h> + +static MALLOC_DEFINE(M_RVV_CTX, "rvv_ctx", "Contexts for user Vector state"); + +static void +vector_enable(void) +{ + uint64_t reg; + + reg = SSTATUS_VS_CLEAN; + + csr_set(sstatus, reg); +} + +static void +vector_disable(void) +{ + uint64_t mask; + + mask = SSTATUS_VS_MASK; + + csr_clear(sstatus, mask); +} + +static void +vector_state_store_common(struct pcb *p, void *datap) +{ + uint64_t vl; + + vector_enable(); + + /* Store vector CSRs. */ + __asm __volatile( + "csrr %0, vstart\n\t" + "csrr %1, vtype\n\t" + "csrr %2, vl\n\t" + "csrr %3, vcsr\n\t" + : "=r" (p->pcb_vstart), "=r" (p->pcb_vtype), + "=r" (p->pcb_vl), "=r" (p->pcb_vcsr) ::); + + if (datap == NULL) + goto done; + + /* Store vector registers. */ + __asm __volatile( + ".option push\n\t" + ".option arch, +zve32x\n\t" + "vsetvli %0, x0, e8, m8, ta, ma\n\t" + "vse8.v v0, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v8, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v16, (%1)\n\t" + "add %1, %1, %0\n\t" + "vse8.v v24, (%1)\n\t" + ".option pop\n\t" + : "=&r" (vl) : "r" (datap) : "memory"); + +done: + vector_disable(); +} + +void +vector_state_store_savectx(struct pcb *p) +{ + + if (!has_vector) + return; + + /* + * savectx() will be called on panic with dumppcb as an argument. + * Save the vector CSR registers, but ignore the vector data. + */ + + vector_state_store_common(p, NULL); +} + +void +vector_state_store(struct thread *td) +{ + struct pcb *p; + void *datap; + + p = td->td_pcb; + + KASSERT(p->pcb_vsaved != NULL, ("VS area is NULL")); + + datap = p->pcb_vsaved; + + vector_state_store_common(p, datap); +} + +void +vector_state_restore(struct thread *td) +{ + struct pcb *p; + void *datap; + uint64_t vl; + + p = td->td_pcb; + + KASSERT(p->pcb_vsaved != NULL, ("VS area is NULL")); + + datap = p->pcb_vsaved; + + vector_enable(); + + /* Restore vector registers. */ + __asm __volatile( + ".option push\n\t" + ".option arch, +zve32x\n\t" + "vsetvli %0, x0, e8, m8, ta, ma\n\t" + "vle8.v v0, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v8, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v16, (%1)\n\t" + "add %1, %1, %0\n\t" + "vle8.v v24, (%1)\n\t" + ".option pop\n\t" + : "=&r" (vl) : "r" (datap) : "memory"); + + /* Restore vector CSRs. */ + __asm __volatile( + ".option push\n\t" + ".option arch, +zve32x\n\t" + "vsetvl x0, %2, %1\n\t" + ".option pop\n\t" + "csrw vstart, %0\n\t" + "csrw vcsr, %3\n\t" + :: "r" (p->pcb_vstart), "r" (p->pcb_vtype), + "r" (p->pcb_vl), "r" (p->pcb_vcsr)); + + vector_disable(); +} + +int +vector_get_size(void) +{ + int len; + + /* RISC-V has 32 vector registers of vlenb size each. */ + + vector_enable(); + len = csr_read(vlenb) * 32; + vector_disable(); + + return (len); +} + +void +vector_state_init(struct thread *td) +{ + struct pcb *p; + int len; + + p = td->td_pcb; + + KASSERT(p->pcb_vsaved == NULL, ("vsaved is already initialized")); + + len = vector_get_size(); + + p->pcb_vsaved = malloc(len, M_RVV_CTX, M_WAITOK | M_ZERO); +} + +void +vector_copy_thread(struct thread *td1, struct thread *td2) +{ + struct pcb *p1; + struct pcb *p2; + int len; + + /* Struct pcb already copied, now init the vector save area. */ + + p1 = td1->td_pcb; + p2 = td2->td_pcb; + p2->pcb_vsaved = NULL; + + vector_state_init(td2); + + len = vector_get_size(); + + memcpy(p2->pcb_vsaved, p1->pcb_vsaved, len); +} + +static void +vector_thread_dtor(void *arg __unused, struct thread *td) +{ + void *datap; + + datap = td->td_pcb->pcb_vsaved; + + free(datap, M_RVV_CTX); +} + +static void +vector_init(const void *dummy __unused) +{ + + EVENTHANDLER_REGISTER(thread_dtor, vector_thread_dtor, NULL, + EVENTHANDLER_PRI_ANY); +} + +SYSINIT(vector, SI_SUB_SMP, SI_ORDER_ANY, vector_init, NULL); diff --git a/sys/riscv/riscv/vm_machdep.c b/sys/riscv/riscv/vm_machdep.c index e538921c9f60..b3e4100ebddb 100644 --- a/sys/riscv/riscv/vm_machdep.c +++ b/sys/riscv/riscv/vm_machdep.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2015-2026 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * Portions of this software were developed by SRI International and the @@ -53,6 +53,7 @@ #include <machine/pcb.h> #include <machine/frame.h> #include <machine/sbi.h> +#include <machine/vector.h> #if __riscv_xlen == 64 #define TP_OFFSET 16 /* sizeof(struct tcb) */ @@ -99,6 +100,14 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) critical_exit(); } + /* Ensure the Vector state is saved before copying the pcb. */ + if ((td1->td_pcb->pcb_vsflags & PCB_VS_STARTED) != 0) { + MPASS(td1 == curthread); + critical_enter(); + vector_state_store(td1); + critical_exit(); + } + pcb2 = td2->td_pcb; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); @@ -120,6 +129,9 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) td2->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; + if ((td1->td_pcb->pcb_vsflags & PCB_VS_STARTED) != 0) + vector_copy_thread(td1, td2); + /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_sstatus_ie = (SSTATUS_SIE); |