diff options
Diffstat (limited to 'sys/cddl/dev')
65 files changed, 4399 insertions, 680 deletions
diff --git a/sys/cddl/dev/dtmalloc/dtmalloc.c b/sys/cddl/dev/dtmalloc/dtmalloc.c index cc6d55eb73dc..e13365bd3007 100644 --- a/sys/cddl/dev/dtmalloc/dtmalloc.c +++ b/sys/cddl/dev/dtmalloc/dtmalloc.c @@ -20,11 +20,8 @@ * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * - * $FreeBSD$ - * */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/conf.h> diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S b/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S index 710232a0d428..3984d12bf67b 100644 --- a/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -72,7 +70,7 @@ uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) - ldrb w0, [x0] + ldtrb w0, [x0] RET END(dtrace_fuword8_nocheck) @@ -81,7 +79,7 @@ uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) - ldrh w0, [x0] + ldtrh w0, [x0] RET END(dtrace_fuword16_nocheck) @@ -90,7 +88,7 @@ uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) - ldr w0, [x0] + ldtr w0, [x0] RET END(dtrace_fuword32_nocheck) @@ -99,7 +97,7 @@ uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) - ldr x0, [x0] + ldtr x0, [x0] RET END(dtrace_fuword64_nocheck) @@ -110,7 +108,8 @@ dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) ENTRY(dtrace_copy) cbz x2, 2f /* If len == 0 then skip loop */ 1: - ldrb w4, [x0], #1 /* Load from uaddr */ + ldtrb w4, [x0] /* Load from uaddr */ + add x0, x0, #1 strb w4, [x1], #1 /* Store in kaddr */ sub x2, x2, #1 /* len-- */ cbnz x2, 1b @@ -126,8 +125,9 @@ XXX: Check for flags? */ ENTRY(dtrace_copystr) cbz x2, 2f /* If len == 0 then skip loop */ - -1: ldrb w4, [x0], #1 /* Load from uaddr */ +1: + ldtrb w4, [x0] /* Load from uaddr */ + add x0, x0, #1 strb w4, [x1], #1 /* Store in kaddr */ cbz w4, 2f /* If == 0 then break */ sub x2, x2, #1 /* len-- */ diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c b/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c index b26b15a58070..34e225f32e81 100644 --- a/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_isa.c @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -50,12 +48,6 @@ #include "regset.h" -/* - * Wee need some reasonable default to prevent backtrace code - * from wandering too far - */ -#define MAX_FUNCTION_SIZE 0x10000 -#define MAX_PROLOGUE_SIZE 0x100 #define MAX_USTACK_DEPTH 2048 uint8_t dtrace_fuword8_nocheck(void *); @@ -139,7 +131,7 @@ dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, break; pc = dtrace_fuword64((void *)(fp + - offsetof(struct arm64_frame, f_retaddr))); + offsetof(struct unwind_state, pc))); fp = dtrace_fuword64((void *)fp); if (fp == oldfp) { @@ -284,12 +276,22 @@ dtrace_getstackdepth(int aframes) } ulong_t -dtrace_getreg(struct trapframe *rp, uint_t reg) +dtrace_getreg(struct trapframe *frame, uint_t reg) { - - printf("IMPLEMENT ME: %s\n", __func__); - - return (0); + switch (reg) { + case REG_X0 ... REG_X29: + return (frame->tf_x[reg]); + case REG_LR: + return (frame->tf_lr); + case REG_SP: + return (frame->tf_sp); + case REG_PC: + return (frame->tf_elr); + default: + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + /* NOTREACHED */ } static int diff --git a/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c b/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c index 74b3bf7ed7d1..b4dae1cba539 100644 --- a/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c +++ b/sys/cddl/dev/dtrace/aarch64/dtrace_subr.c @@ -19,26 +19,22 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/systm.h> -#include <sys/types.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/kmem.h> +#include <sys/proc.h> #include <sys/smp.h> #include <sys/dtrace_impl.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> #include <machine/armreg.h> #include <machine/clock.h> #include <machine/frame.h> @@ -65,17 +61,20 @@ dtrace_invop_hdlr_t *dtrace_invop_hdlr; int dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) { + struct thread *td; dtrace_invop_hdlr_t *hdlr; int rval; + rval = 0; + td = curthread; + td->t_dtrace_trapframe = frame; for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) - return (rval); - - return (0); + break; + td->t_dtrace_trapframe = NULL; + return (rval); } - void dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t)) { @@ -207,7 +206,7 @@ dtrace_trap(struct trapframe *frame, u_int type) case EXCP_DATA_ABORT: /* Flag a bad address. */ cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; - cpu_core[curcpu].cpuc_dtrace_illval = 0; + cpu_core[curcpu].cpuc_dtrace_illval = frame->tf_far; /* * Offset the instruction pointer to the instruction diff --git a/sys/cddl/dev/dtrace/aarch64/instr_size.c b/sys/cddl/dev/dtrace/aarch64/instr_size.c new file mode 100644 index 000000000000..35964c0f0f78 --- /dev/null +++ b/sys/cddl/dev/dtrace/aarch64/instr_size.c @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright 2023 Christos Margiolis <christos@FreeBSD.org> + */ + +#include <sys/types.h> +#include <sys/dtrace.h> + +int +dtrace_instr_size(uint8_t *instr __unused) +{ + return (INSN_SIZE); +} diff --git a/sys/cddl/dev/dtrace/aarch64/regset.h b/sys/cddl/dev/dtrace/aarch64/regset.h index f99b48f8354f..b20e382e9ce7 100644 --- a/sys/cddl/dev/dtrace/aarch64/regset.h +++ b/sys/cddl/dev/dtrace/aarch64/regset.h @@ -19,7 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -35,14 +34,46 @@ #define _REGSET_H /* - * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" */ #ifdef __cplusplus extern "C" { #endif -/* Place here */ +#define REG_X0 0 +#define REG_X1 1 +#define REG_X2 2 +#define REG_X3 3 +#define REG_X4 4 +#define REG_X5 5 +#define REG_X6 6 +#define REG_X7 7 +#define REG_X8 8 +#define REG_X9 9 +#define REG_X10 10 +#define REG_X11 11 +#define REG_X12 12 +#define REG_X13 13 +#define REG_X14 14 +#define REG_X15 15 +#define REG_X16 16 +#define REG_X17 17 +#define REG_X18 18 +#define REG_X19 19 +#define REG_X20 20 +#define REG_X21 21 +#define REG_X22 22 +#define REG_X23 23 +#define REG_X24 24 +#define REG_X25 25 +#define REG_X26 26 +#define REG_X27 27 +#define REG_X28 28 +#define REG_X29 29 +#define REG_FP REG_X29 +#define REG_LR 30 +#define REG_SP 31 +#define REG_PC 32 #ifdef __cplusplus } diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_asm.S b/sys/cddl/dev/dtrace/amd64/dtrace_asm.S index 9a83c87dda02..0c8cd9a83d01 100644 --- a/sys/cddl/dev/dtrace/amd64/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/amd64/dtrace_asm.S @@ -20,8 +20,6 @@ * * Portions Copyright 2008 John Birrell <jb@freebsd.org> * - * $FreeBSD$ - * */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. @@ -58,6 +56,8 @@ swapgs; \ 1: addq $TF_RIP,%rsp; +.globl dtrace_invop_callsite +.type dtrace_invop_callsite,@function ENTRY(dtrace_invop_start) @@ -69,11 +69,22 @@ movq TF_RIP(%rsp), %rdi decq %rdi movq %rsp, %rsi - movq TF_RAX(%rsp), %rdx + + /* + * Allocate some scratch space to let the invop handler return a value. + * This is needed when emulating "call" instructions. + */ + subq $16, %rsp + movq %rsp, %rdx + call dtrace_invop - ENTRY(dtrace_invop_callsite) +dtrace_invop_callsite: + addq $16, %rsp + cmpl $DTRACE_INVOP_PUSHL_EBP, %eax je bp_push + cmpl $DTRACE_INVOP_CALL, %eax + je bp_call cmpl $DTRACE_INVOP_LEAVE, %eax je bp_leave cmpl $DTRACE_INVOP_NOP, %eax @@ -109,6 +120,40 @@ bp_push: iretq /* return from interrupt */ /*NOTREACHED*/ +bp_call: + /* + * Emulate a "call" instruction. The invop handler must have already + * updated the saved copy of %rip in the register set. It's our job to + * pull the hardware-saved registers down to make space for the return + * address, which is provided by the invop handler in our scratch + * space. + */ + INTR_POP + subq $16, %rsp /* make room for %rbp */ + pushq %rax /* push temp */ + pushq %rbx /* push temp */ + + movq 32(%rsp), %rax /* load calling RIP */ + movq %rax, 16(%rsp) /* store calling RIP */ + movq 40(%rsp), %rax /* load calling CS */ + movq %rax, 24(%rsp) /* store calling CS */ + movq 48(%rsp), %rax /* load calling RFLAGS */ + movq %rax, 32(%rsp) /* store calling RFLAGS */ + movq 56(%rsp), %rax /* load calling RSP */ + subq $8, %rax /* make room for return address */ + movq %rax, 40(%rsp) /* store calling RSP */ + movq 64(%rsp), %rax /* load calling SS */ + movq %rax, 48(%rsp) /* store calling SS */ + + movq -(TF_RIP - 16)(%rsp), %rax /* load return address */ + movq 40(%rsp), %rbx /* reload calling RSP */ + movq %rax, (%rbx) /* store return address */ + + popq %rbx /* pop temp */ + popq %rax /* pop temp */ + iretq /* return from interrupt */ + /*NOTREACHED*/ + bp_leave: /* * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c index 71b448a99c1c..0b7162998536 100644 --- a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -476,7 +474,7 @@ dtrace_getstackdepth(int aframes) } ulong_t -dtrace_getreg(struct trapframe *rp, uint_t reg) +dtrace_getreg(struct trapframe *frame, uint_t reg) { /* This table is dependent on reg.d. */ int regmap[] = { @@ -501,11 +499,7 @@ dtrace_getreg(struct trapframe *rp, uint_t reg) REG_SS /* 18 SS */ }; -#ifdef illumos - if (reg <= SS) { -#else /* !illumos */ if (reg <= GS) { -#endif if (reg >= sizeof (regmap) / sizeof (int)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); @@ -514,66 +508,62 @@ dtrace_getreg(struct trapframe *rp, uint_t reg) reg = regmap[reg]; } else { /* This is dependent on reg.d. */ -#ifdef illumos - reg -= SS + 1; -#else /* !illumos */ reg -= GS + 1; -#endif } switch (reg) { case REG_RDI: - return (rp->tf_rdi); + return (frame->tf_rdi); case REG_RSI: - return (rp->tf_rsi); + return (frame->tf_rsi); case REG_RDX: - return (rp->tf_rdx); + return (frame->tf_rdx); case REG_RCX: - return (rp->tf_rcx); + return (frame->tf_rcx); case REG_R8: - return (rp->tf_r8); + return (frame->tf_r8); case REG_R9: - return (rp->tf_r9); + return (frame->tf_r9); case REG_RAX: - return (rp->tf_rax); + return (frame->tf_rax); case REG_RBX: - return (rp->tf_rbx); + return (frame->tf_rbx); case REG_RBP: - return (rp->tf_rbp); + return (frame->tf_rbp); case REG_R10: - return (rp->tf_r10); + return (frame->tf_r10); case REG_R11: - return (rp->tf_r11); + return (frame->tf_r11); case REG_R12: - return (rp->tf_r12); + return (frame->tf_r12); case REG_R13: - return (rp->tf_r13); + return (frame->tf_r13); case REG_R14: - return (rp->tf_r14); + return (frame->tf_r14); case REG_R15: - return (rp->tf_r15); + return (frame->tf_r15); case REG_DS: - return (rp->tf_ds); + return (frame->tf_ds); case REG_ES: - return (rp->tf_es); + return (frame->tf_es); case REG_FS: - return (rp->tf_fs); + return (frame->tf_fs); case REG_GS: - return (rp->tf_gs); + return (frame->tf_gs); case REG_TRAPNO: - return (rp->tf_trapno); + return (frame->tf_trapno); case REG_ERR: - return (rp->tf_err); + return (frame->tf_err); case REG_RIP: - return (rp->tf_rip); + return (frame->tf_rip); case REG_CS: - return (rp->tf_cs); + return (frame->tf_cs); case REG_SS: - return (rp->tf_ss); + return (frame->tf_ss); case REG_RFL: - return (rp->tf_rflags); + return (frame->tf_rflags); case REG_RSP: - return (rp->tf_rsp); + return (frame->tf_rsp); default: DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_subr.c b/sys/cddl/dev/dtrace/amd64/dtrace_subr.c index 4f9d9995cbab..09b820241e50 100644 --- a/sys/cddl/dev/dtrace/amd64/dtrace_subr.c +++ b/sys/cddl/dev/dtrace/amd64/dtrace_subr.c @@ -19,8 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -33,12 +31,13 @@ #include <sys/param.h> #include <sys/systm.h> -#include <sys/types.h> #include <sys/kernel.h> #include <sys/malloc.h> +#include <sys/proc.h> #include <sys/smp.h> #include <sys/dtrace_impl.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> #include <machine/clock.h> #include <machine/cpufunc.h> #include <machine/frame.h> @@ -50,7 +49,7 @@ extern void dtrace_getnanotime(struct timespec *tsp); extern int (*dtrace_invop_jump_addr)(struct trapframe *); -int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t); +int dtrace_invop(uintptr_t, struct trapframe *, void **); int dtrace_invop_start(struct trapframe *frame); void dtrace_invop_init(void); void dtrace_invop_uninit(void); @@ -63,16 +62,22 @@ typedef struct dtrace_invop_hdlr { dtrace_invop_hdlr_t *dtrace_invop_hdlr; int -dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) +dtrace_invop(uintptr_t addr, struct trapframe *frame, void **scratch) { + struct thread *td; dtrace_invop_hdlr_t *hdlr; int rval; - for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) - if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) - return (rval); - - return (0); + td = curthread; + td->t_dtrace_trapframe = frame; + rval = 0; + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) { + rval = hdlr->dtih_func(addr, frame, (uintptr_t)scratch); + if (rval != 0) + break; + } + td->t_dtrace_trapframe = NULL; + return (rval); } void @@ -278,7 +283,6 @@ dtrace_gethrtime_init_cpu(void *arg) hst_cpu_tsc = rdtsc(); } -#ifdef EARLY_AP_STARTUP static void dtrace_gethrtime_init(void *arg) { @@ -286,16 +290,6 @@ dtrace_gethrtime_init(void *arg) uint64_t tsc_f; cpuset_t map; int i; -#else -/* - * Get the frequency and scale factor as early as possible so that they can be - * used for boot-time tracing. - */ -static void -dtrace_gethrtime_init_early(void *arg) -{ - uint64_t tsc_f; -#endif /* * Get TSC frequency known at this moment. @@ -324,18 +318,6 @@ dtrace_gethrtime_init_early(void *arg) * (terahertz) values; */ nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; -#ifndef EARLY_AP_STARTUP -} -SYSINIT(dtrace_gethrtime_init_early, SI_SUB_CPU, SI_ORDER_ANY, - dtrace_gethrtime_init_early, NULL); - -static void -dtrace_gethrtime_init(void *arg) -{ - struct pcpu *pc; - cpuset_t map; - int i; -#endif if (vm_guest != VM_GUEST_NO) return; @@ -359,13 +341,8 @@ dtrace_gethrtime_init(void *arg) } sched_unpin(); } -#ifdef EARLY_AP_STARTUP SYSINIT(dtrace_gethrtime_init, SI_SUB_DTRACE, SI_ORDER_ANY, dtrace_gethrtime_init, NULL); -#else -SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, - NULL); -#endif /* * DTrace needs a high resolution time function which can @@ -442,7 +419,7 @@ dtrace_trap(struct trapframe *frame, u_int type) * Offset the instruction pointer to the instruction * following the one causing the fault. */ - frame->tf_rip += dtrace_instr_size((u_char *) frame->tf_rip); + frame->tf_rip += dtrace_instr_size((uint8_t *) frame->tf_rip); return (1); /* Page fault. */ case T_PAGEFLT: @@ -454,7 +431,7 @@ dtrace_trap(struct trapframe *frame, u_int type) * Offset the instruction pointer to the instruction * following the one causing the fault. */ - frame->tf_rip += dtrace_instr_size((u_char *) frame->tf_rip); + frame->tf_rip += dtrace_instr_size((uint8_t *) frame->tf_rip); return (1); default: /* Handle all other traps in the usual way. */ diff --git a/sys/cddl/dev/dtrace/arm/dtrace_asm.S b/sys/cddl/dev/dtrace/arm/dtrace_asm.S index 1b9996bb133a..b0eba6f1ed9f 100644 --- a/sys/cddl/dev/dtrace/arm/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/arm/dtrace_asm.S @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. diff --git a/sys/cddl/dev/dtrace/arm/dtrace_isa.c b/sys/cddl/dev/dtrace/arm/dtrace_isa.c index ede352e6b873..c3783b77c2d4 100644 --- a/sys/cddl/dev/dtrace/arm/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/arm/dtrace_isa.c @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -50,14 +48,6 @@ #include "regset.h" -/* - * Wee need some reasonable default to prevent backtrace code - * from wandering too far - */ -#define MAX_FUNCTION_SIZE 0x10000 -#define MAX_PROLOGUE_SIZE 0x100 - - uint8_t dtrace_fuword8_nocheck(void *); uint16_t dtrace_fuword16_nocheck(void *); uint32_t dtrace_fuword32_nocheck(void *); @@ -165,7 +155,7 @@ dtrace_getstackdepth(int aframes) } ulong_t -dtrace_getreg(struct trapframe *rp, uint_t reg) +dtrace_getreg(struct trapframe *frame, uint_t reg) { printf("IMPLEMENT ME: %s\n", __func__); diff --git a/sys/cddl/dev/dtrace/arm/dtrace_subr.c b/sys/cddl/dev/dtrace/arm/dtrace_subr.c index e98a9ded5442..bb42044aa477 100644 --- a/sys/cddl/dev/dtrace/arm/dtrace_subr.c +++ b/sys/cddl/dev/dtrace/arm/dtrace_subr.c @@ -19,26 +19,22 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/systm.h> -#include <sys/types.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/kmem.h> +#include <sys/proc.h> #include <sys/smp.h> #include <sys/dtrace_impl.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> #include <machine/armreg.h> #include <machine/clock.h> #include <machine/frame.h> @@ -70,14 +66,18 @@ dtrace_invop_hdlr_t *dtrace_invop_hdlr; int dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) { + struct thread *td; dtrace_invop_hdlr_t *hdlr; int rval; + rval = 0; + td = curthread; + td->t_dtrace_trapframe = frame; for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) - return (rval); - - return (0); + break; + td->t_dtrace_trapframe = NULL; + return (rval); } @@ -171,7 +171,7 @@ dtrace_sync(void) * Returns nanoseconds since boot. */ uint64_t -dtrace_gethrtime() +dtrace_gethrtime(void) { struct timespec curtime; diff --git a/sys/cddl/dev/dtrace/arm/regset.h b/sys/cddl/dev/dtrace/arm/regset.h index ce9e97ea7a09..4bbea0e10832 100644 --- a/sys/cddl/dev/dtrace/arm/regset.h +++ b/sys/cddl/dev/dtrace/arm/regset.h @@ -19,7 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -35,7 +34,6 @@ #define _REGSET_H /* - * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" */ #ifdef __cplusplus diff --git a/sys/cddl/dev/dtrace/dtrace_anon.c b/sys/cddl/dev/dtrace/dtrace_anon.c index b81ec5be3f11..8c525f528ed9 100644 --- a/sys/cddl/dev/dtrace/dtrace_anon.c +++ b/sys/cddl/dev/dtrace/dtrace_anon.c @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* diff --git a/sys/cddl/dev/dtrace/dtrace_cddl.h b/sys/cddl/dev/dtrace/dtrace_cddl.h index dd62af632e79..95317955c102 100644 --- a/sys/cddl/dev/dtrace/dtrace_cddl.h +++ b/sys/cddl/dev/dtrace/dtrace_cddl.h @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef _DTRACE_CDDL_H_ @@ -87,6 +85,8 @@ typedef struct kdtrace_thread { void *td_dtrace_sscr; /* Saved scratch space location. */ void *td_systrace_args; /* syscall probe arguments. */ uint64_t td_fasttrap_tp_gen; /* Tracepoint hash table gen. */ + struct trapframe *td_dtrace_trapframe; /* Trap frame from invop. */ + void *td_kinst_tramp; } kdtrace_thread_t; /* @@ -115,6 +115,8 @@ typedef struct kdtrace_thread { #define t_dtrace_sscr td_dtrace->td_dtrace_sscr #define t_dtrace_systrace_args td_dtrace->td_systrace_args #define t_fasttrap_tp_gen td_dtrace->td_fasttrap_tp_gen +#define t_dtrace_trapframe td_dtrace->td_dtrace_trapframe +#define t_kinst_tramp td_dtrace->td_kinst_tramp #define p_dtrace_helpers p_dtrace->p_dtrace_helpers #define p_dtrace_count p_dtrace->p_dtrace_count #define p_dtrace_probes p_dtrace->p_dtrace_probes diff --git a/sys/cddl/dev/dtrace/dtrace_debug.c b/sys/cddl/dev/dtrace/dtrace_debug.c index 5edd2fd603e1..48b837894165 100644 --- a/sys/cddl/dev/dtrace/dtrace_debug.c +++ b/sys/cddl/dev/dtrace/dtrace_debug.c @@ -25,8 +25,6 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. * - * $FreeBSD$ - * */ #ifdef DEBUG diff --git a/sys/cddl/dev/dtrace/dtrace_hacks.c b/sys/cddl/dev/dtrace/dtrace_hacks.c index 3f8997382c75..f69489a657f7 100644 --- a/sys/cddl/dev/dtrace/dtrace_hacks.c +++ b/sys/cddl/dev/dtrace/dtrace_hacks.c @@ -1,4 +1,3 @@ -/* $FreeBSD$ */ /* XXX Hacks.... */ dtrace_cacheid_t dtrace_predcache_id; diff --git a/sys/cddl/dev/dtrace/dtrace_ioctl.c b/sys/cddl/dev/dtrace/dtrace_ioctl.c index da36ee73897c..be7fff170166 100644 --- a/sys/cddl/dev/dtrace/dtrace_ioctl.c +++ b/sys/cddl/dev/dtrace/dtrace_ioctl.c @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ static int dtrace_verbose_ioctl; diff --git a/sys/cddl/dev/dtrace/dtrace_load.c b/sys/cddl/dev/dtrace/dtrace_load.c index 232d7df6999c..ef2d29e143f8 100644 --- a/sys/cddl/dev/dtrace/dtrace_load.c +++ b/sys/cddl/dev/dtrace/dtrace_load.c @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef EARLY_AP_STARTUP diff --git a/sys/cddl/dev/dtrace/dtrace_modevent.c b/sys/cddl/dev/dtrace/dtrace_modevent.c index 8d318532dee9..aeb237be31a0 100644 --- a/sys/cddl/dev/dtrace/dtrace_modevent.c +++ b/sys/cddl/dev/dtrace/dtrace_modevent.c @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* ARGSUSED */ diff --git a/sys/cddl/dev/dtrace/dtrace_sysctl.c b/sys/cddl/dev/dtrace/dtrace_sysctl.c index db2c44db0249..d617fbd93973 100644 --- a/sys/cddl/dev/dtrace/dtrace_sysctl.c +++ b/sys/cddl/dev/dtrace/dtrace_sysctl.c @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* Report registered DTrace providers. */ diff --git a/sys/cddl/dev/dtrace/dtrace_test.c b/sys/cddl/dev/dtrace/dtrace_test.c index eccfc1a4dfd4..92ba3a36a24e 100644 --- a/sys/cddl/dev/dtrace/dtrace_test.c +++ b/sys/cddl/dev/dtrace/dtrace_test.c @@ -22,10 +22,8 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD$ - * */ -#include <sys/cdefs.h> + #include <sys/types.h> #include <sys/param.h> #include <sys/systm.h> diff --git a/sys/cddl/dev/dtrace/dtrace_unload.c b/sys/cddl/dev/dtrace/dtrace_unload.c index 1dfaa774e99f..48261787f374 100644 --- a/sys/cddl/dev/dtrace/dtrace_unload.c +++ b/sys/cddl/dev/dtrace/dtrace_unload.c @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ static int diff --git a/sys/cddl/dev/dtrace/dtrace_vtime.c b/sys/cddl/dev/dtrace/dtrace_vtime.c index a3fa7f77287a..c766ba910691 100644 --- a/sys/cddl/dev/dtrace/dtrace_vtime.c +++ b/sys/cddl/dev/dtrace/dtrace_vtime.c @@ -17,8 +17,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* diff --git a/sys/cddl/dev/dtrace/i386/dtrace_asm.S b/sys/cddl/dev/dtrace/i386/dtrace_asm.S index e8767c74efc0..af10064903e7 100644 --- a/sys/cddl/dev/dtrace/i386/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/i386/dtrace_asm.S @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. diff --git a/sys/cddl/dev/dtrace/i386/dtrace_isa.c b/sys/cddl/dev/dtrace/i386/dtrace_isa.c index 3c8602c1f7be..64c8de2a8d3a 100644 --- a/sys/cddl/dev/dtrace/i386/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/i386/dtrace_isa.c @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -498,7 +496,7 @@ dtrace_getstackdepth(int aframes) } ulong_t -dtrace_getreg(struct trapframe *rp, uint_t reg) +dtrace_getreg(struct trapframe *frame, uint_t reg) { struct pcb *pcb; int regmap[] = { /* Order is dependent on reg.d */ @@ -543,41 +541,41 @@ dtrace_getreg(struct trapframe *rp, uint_t reg) } return (pcb->pcb_gs); case REG_FS: - return (rp->tf_fs); + return (frame->tf_fs); case REG_ES: - return (rp->tf_es); + return (frame->tf_es); case REG_DS: - return (rp->tf_ds); + return (frame->tf_ds); case REG_RDI: - return (rp->tf_edi); + return (frame->tf_edi); case REG_RSI: - return (rp->tf_esi); + return (frame->tf_esi); case REG_RBP: - return (rp->tf_ebp); + return (frame->tf_ebp); case REG_RSP: - return (rp->tf_isp); + return (frame->tf_isp); case REG_RBX: - return (rp->tf_ebx); + return (frame->tf_ebx); case REG_RCX: - return (rp->tf_ecx); + return (frame->tf_ecx); case REG_RAX: - return (rp->tf_eax); + return (frame->tf_eax); case REG_TRAPNO: - return (rp->tf_trapno); + return (frame->tf_trapno); case REG_ERR: - return (rp->tf_err); + return (frame->tf_err); case REG_RIP: - return (rp->tf_eip); + return (frame->tf_eip); case REG_CS: - return (rp->tf_cs); + return (frame->tf_cs); case REG_RFL: - return (rp->tf_eflags); + return (frame->tf_eflags); #if 0 case REG_RSP: - return (rp->tf_esp); + return (frame->tf_esp); #endif case REG_SS: - return (rp->tf_ss); + return (frame->tf_ss); default: DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); diff --git a/sys/cddl/dev/dtrace/i386/dtrace_subr.c b/sys/cddl/dev/dtrace/i386/dtrace_subr.c index 37cc7601bef5..026581f5a899 100644 --- a/sys/cddl/dev/dtrace/i386/dtrace_subr.c +++ b/sys/cddl/dev/dtrace/i386/dtrace_subr.c @@ -19,8 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -33,14 +31,15 @@ #include <sys/param.h> #include <sys/systm.h> -#include <sys/types.h> #include <sys/cpuset.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/kmem.h> +#include <sys/proc.h> #include <sys/smp.h> #include <sys/dtrace_impl.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> #include <machine/clock.h> #include <machine/cpufunc.h> #include <machine/frame.h> @@ -68,14 +67,18 @@ dtrace_invop_hdlr_t *dtrace_invop_hdlr; int dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax) { + struct thread *td; dtrace_invop_hdlr_t *hdlr; int rval; + rval = 0; + td = curthread; + td->t_dtrace_trapframe = frame; for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0) - return (rval); - - return (0); + break; + td->t_dtrace_trapframe = NULL; + return (rval); } void @@ -280,7 +283,6 @@ dtrace_gethrtime_init_cpu(void *arg) hst_cpu_tsc = rdtsc(); } -#ifdef EARLY_AP_STARTUP static void dtrace_gethrtime_init(void *arg) { @@ -288,16 +290,6 @@ dtrace_gethrtime_init(void *arg) uint64_t tsc_f; cpuset_t map; int i; -#else -/* - * Get the frequency and scale factor as early as possible so that they can be - * used for boot-time tracing. - */ -static void -dtrace_gethrtime_init_early(void *arg) -{ - uint64_t tsc_f; -#endif /* * Get TSC frequency known at this moment. @@ -326,18 +318,6 @@ dtrace_gethrtime_init_early(void *arg) * (terahertz) values; */ nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f; -#ifndef EARLY_AP_STARTUP -} -SYSINIT(dtrace_gethrtime_init_early, SI_SUB_CPU, SI_ORDER_ANY, - dtrace_gethrtime_init_early, NULL); - -static void -dtrace_gethrtime_init(void *arg) -{ - cpuset_t map; - struct pcpu *pc; - int i; -#endif if (vm_guest != VM_GUEST_NO) return; @@ -361,13 +341,8 @@ dtrace_gethrtime_init(void *arg) } sched_unpin(); } -#ifdef EARLY_AP_STARTUP SYSINIT(dtrace_gethrtime_init, SI_SUB_DTRACE, SI_ORDER_ANY, dtrace_gethrtime_init, NULL); -#else -SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, - NULL); -#endif /* * DTrace needs a high resolution time function which can @@ -444,7 +419,7 @@ dtrace_trap(struct trapframe *frame, u_int type) * Offset the instruction pointer to the instruction * following the one causing the fault. */ - frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip); + frame->tf_eip += dtrace_instr_size((uint8_t *) frame->tf_eip); return (1); /* Page fault. */ case T_PAGEFLT: @@ -456,7 +431,7 @@ dtrace_trap(struct trapframe *frame, u_int type) * Offset the instruction pointer to the instruction * following the one causing the fault. */ - frame->tf_eip += dtrace_instr_size((u_char *) frame->tf_eip); + frame->tf_eip += dtrace_instr_size((uint8_t *) frame->tf_eip); return (1); default: /* Handle all other traps in the usual way. */ diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S b/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S index 3371068896db..807492dc199d 100644 --- a/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S @@ -20,8 +20,6 @@ * CDDL HEADER END * * Portions Copyright 2012,2013 Justin Hibbits <jhibbits@freebsd.org> - * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c index cce1c907b5d8..7185a01c125d 100644 --- a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c @@ -20,8 +20,6 @@ * CDDL HEADER END * * Portions Copyright 2012,2013 Justin Hibbits <jhibbits@freebsd.org> - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -97,15 +95,18 @@ dtrace_sp_inkernel(uintptr_t sp) } static __inline void -dtrace_next_sp_pc(uintptr_t sp, uintptr_t *nsp, uintptr_t *pc) +dtrace_next_sp_pc(uintptr_t sp, uintptr_t *nsp, uintptr_t *pc, uintptr_t *lr) { vm_offset_t callpc; struct trapframe *frame; + if (lr != 0 && *lr != 0) + callpc = *lr; + else #ifdef __powerpc64__ - callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); + callpc = *(vm_offset_t *)(sp + RETURN_OFFSET64); #else - callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); + callpc = *(vm_offset_t *)(sp + RETURN_OFFSET); #endif /* @@ -121,6 +122,8 @@ dtrace_next_sp_pc(uintptr_t sp, uintptr_t *nsp, uintptr_t *pc) *nsp = frame->fixreg[1]; if (pc != NULL) *pc = frame->srr0; + if (lr != NULL) + *lr = frame->lr; return; } @@ -128,6 +131,9 @@ dtrace_next_sp_pc(uintptr_t sp, uintptr_t *nsp, uintptr_t *pc) *nsp = *(uintptr_t *)sp; if (pc != NULL) *pc = callpc; + /* lr is only valid for trap frames */ + if (lr != NULL) + *lr = 0; } void @@ -135,7 +141,7 @@ dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, uint32_t *intrpc) { int depth = 0; - uintptr_t osp, sp; + uintptr_t osp, sp, lr = 0; vm_offset_t callpc; pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; @@ -154,7 +160,7 @@ dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, if (!dtrace_sp_inkernel(sp)) break; osp = sp; - dtrace_next_sp_pc(osp, &sp, &callpc); + dtrace_next_sp_pc(osp, &sp, &callpc, &lr); if (aframes > 0) { aframes--; @@ -513,7 +519,7 @@ dtrace_getstackdepth(int aframes) depth++; osp = sp; - dtrace_next_sp_pc(sp, &sp, NULL); + dtrace_next_sp_pc(sp, &sp, NULL, NULL); } if (depth < aframes) return (0); @@ -522,26 +528,26 @@ dtrace_getstackdepth(int aframes) } ulong_t -dtrace_getreg(struct trapframe *rp, uint_t reg) +dtrace_getreg(struct trapframe *frame, uint_t reg) { if (reg < 32) - return (rp->fixreg[reg]); + return (frame->fixreg[reg]); switch (reg) { case 32: - return (rp->lr); + return (frame->lr); case 33: - return (rp->cr); + return (frame->cr); case 34: - return (rp->xer); + return (frame->xer); case 35: - return (rp->ctr); + return (frame->ctr); case 36: - return (rp->srr0); + return (frame->srr0); case 37: - return (rp->srr1); + return (frame->srr1); case 38: - return (rp->exc); + return (frame->exc); default: DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c b/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c index ed171f1135e9..5dd083310e6f 100644 --- a/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c +++ b/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c @@ -19,26 +19,22 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/systm.h> -#include <sys/types.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/kmem.h> +#include <sys/proc.h> #include <sys/smp.h> #include <sys/dtrace_impl.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> #include <machine/clock.h> #include <machine/frame.h> #include <machine/trap.h> @@ -65,14 +61,18 @@ dtrace_invop_hdlr_t *dtrace_invop_hdlr; int dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t arg0) { + struct thread *td; dtrace_invop_hdlr_t *hdlr; int rval; + rval = 0; + td = curthread; + td->t_dtrace_trapframe = frame; for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) if ((rval = hdlr->dtih_func(addr, frame, arg0)) != 0) - return (rval); - - return (0); + break; + td->t_dtrace_trapframe = NULL; + return (rval); } void @@ -234,7 +234,7 @@ SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, * Returns nanoseconds since boot. */ uint64_t -dtrace_gethrtime() +dtrace_gethrtime(void) { uint64_t timebase; uint32_t lo; diff --git a/sys/cddl/dev/dtrace/powerpc/regset.h b/sys/cddl/dev/dtrace/powerpc/regset.h index 64973885fc96..b47bd54e081e 100644 --- a/sys/cddl/dev/dtrace/powerpc/regset.h +++ b/sys/cddl/dev/dtrace/powerpc/regset.h @@ -19,7 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -35,7 +34,6 @@ #define _REGSET_H /* - * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" */ #ifdef __cplusplus diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_asm.S b/sys/cddl/dev/dtrace/riscv/dtrace_asm.S index eeaf64061675..4d7dc0deacf1 100644 --- a/sys/cddl/dev/dtrace/riscv/dtrace_asm.S +++ b/sys/cddl/dev/dtrace/riscv/dtrace_asm.S @@ -20,8 +20,6 @@ * CDDL HEADER END * * Portions Copyright 2016 Ruslan Bukin <br@bsdpad.com> - * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -74,7 +72,9 @@ uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) + ENTER_USER_ACCESS(t0) lb a0, 0(a0) + EXIT_USER_ACCESS(t0) RET END(dtrace_fuword8_nocheck) @@ -83,7 +83,9 @@ uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) + ENTER_USER_ACCESS(t0) lh a0, 0(a0) + EXIT_USER_ACCESS(t0) RET END(dtrace_fuword16_nocheck) @@ -92,7 +94,9 @@ uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) + ENTER_USER_ACCESS(t0) lw a0, 0(a0) + EXIT_USER_ACCESS(t0) RET END(dtrace_fuword32_nocheck) @@ -101,7 +105,9 @@ uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) + ENTER_USER_ACCESS(t0) ld a0, 0(a0) + EXIT_USER_ACCESS(t0) RET END(dtrace_fuword64_nocheck) @@ -111,6 +117,7 @@ dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ ENTRY(dtrace_copy) beqz a2, 2f /* If len == 0 then skip loop */ + ENTER_USER_ACCESS(t0) 1: lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 @@ -118,6 +125,7 @@ ENTRY(dtrace_copy) addi a1, a1, 1 addi a2, a2, -1 /* len-- */ bnez a2, 1b + EXIT_USER_ACCESS(t0) 2: RET END(dtrace_copy) @@ -129,7 +137,9 @@ dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, XXX: Check for flags? */ ENTRY(dtrace_copystr) - beqz a2, 2f /* If len == 0 then skip loop */ + beqz a2, 3f /* If len == 0 then skip loop */ + ENTER_USER_ACCESS(t0) +1: lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ @@ -138,6 +148,8 @@ ENTRY(dtrace_copystr) addi a2, a2, -1 /* len-- */ bnez a2, 1b 2: + EXIT_USER_ACCESS(t0) +3: RET END(dtrace_copystr) diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_isa.c b/sys/cddl/dev/dtrace/riscv/dtrace_isa.c index d42299abcd35..87f52b809dfa 100644 --- a/sys/cddl/dev/dtrace/riscv/dtrace_isa.c +++ b/sys/cddl/dev/dtrace/riscv/dtrace_isa.c @@ -20,8 +20,6 @@ * CDDL HEADER END * * Portions Copyright 2016 Ruslan Bukin <br@bsdpad.com> - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -37,6 +35,8 @@ #include <machine/frame.h> #include <machine/md_var.h> +#include <machine/encoding.h> +#include <machine/riscvreg.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -52,12 +52,6 @@ #include "regset.h" -/* - * Wee need some reasonable default to prevent backtrace code - * from wandering too far - */ -#define MAX_FUNCTION_SIZE 0x10000 -#define MAX_PROLOGUE_SIZE 0x100 #define MAX_USTACK_DEPTH 2048 uint8_t dtrace_fuword8_nocheck(void *); @@ -65,47 +59,61 @@ uint16_t dtrace_fuword16_nocheck(void *); uint32_t dtrace_fuword32_nocheck(void *); uint64_t dtrace_fuword64_nocheck(void *); +int dtrace_match_opcode(uint32_t, int, int); +int dtrace_instr_sdsp(uint32_t **); +int dtrace_instr_ret(uint32_t **); +int dtrace_instr_c_sdsp(uint32_t **); +int dtrace_instr_c_ret(uint32_t **); + void dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, uint32_t *intrpc) { struct unwind_state state; - int scp_offset; + uintptr_t caller; register_t sp; + int scp_offset; int depth; depth = 0; + caller = solaris_cpu[curcpu].cpu_dtrace_caller; if (intrpc != 0) { - pcstack[depth++] = (pc_t) intrpc; + pcstack[depth++] = (pc_t)intrpc; } - aframes++; - + /* + * Construct the unwind state, starting from this function. This frame, + * and 'aframes' others will be skipped. + */ __asm __volatile("mv %0, sp" : "=&r" (sp)); state.fp = (uintptr_t)__builtin_frame_address(0); - state.sp = sp; + state.sp = (uintptr_t)sp; state.pc = (uintptr_t)dtrace_getpcstack; while (depth < pcstack_limit) { if (!unwind_frame(curthread, &state)) break; - if (!INKERNEL(state.pc) || !INKERNEL(state.fp)) + if (!INKERNEL(state.pc) || !kstack_contains(curthread, + (vm_offset_t)state.fp, sizeof(uintptr_t))) break; - /* - * NB: Unlike some other architectures, we don't need to - * explicitly insert cpu_dtrace_caller as it appears in the - * normal kernel stack trace rather than a special trap frame. - */ if (aframes > 0) { aframes--; + + /* + * fbt_invop() records the return address at the time + * the FBT probe fires. We need to insert this into the + * backtrace manually, since the stack frame state at + * the time of the probe does not capture it. + */ + if (aframes == 0 && caller != 0) + pcstack[depth++] = caller; } else { pcstack[depth++] = state.pc; } - } for (; depth < pcstack_limit; depth++) { @@ -148,9 +156,8 @@ dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc, if (fp == 0) break; - pc = dtrace_fuword64((void *)(fp + - offsetof(struct riscv_frame, f_retaddr))); - fp = dtrace_fuword64((void *)fp); + pc = dtrace_fuword64((void *)(fp - 1 * sizeof(uint64_t))); + fp = dtrace_fuword64((void *)(fp - 2 * sizeof(uint64_t))); if (fp == oldfp) { *flags |= CPU_DTRACE_BADSTACK; @@ -168,7 +175,7 @@ dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) { volatile uint16_t *flags; struct trapframe *tf; - uintptr_t pc, sp, fp; + uintptr_t pc, fp; proc_t *p; int n; @@ -194,7 +201,6 @@ dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) return; pc = tf->tf_sepc; - sp = tf->tf_sp; fp = tf->tf_s[0]; if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { @@ -206,7 +212,6 @@ dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) * at the current stack pointer address since the call * instruction puts it there right before the branch. */ - *pcstack++ = (uint64_t)pc; pcstack_limit--; if (pcstack_limit <= 0) @@ -230,8 +235,33 @@ zero: int dtrace_getustackdepth(void) { + struct trapframe *tf; + uintptr_t pc, fp; + int n = 0; - printf("IMPLEMENT ME: %s\n", __func__); + if (curproc == NULL || (tf = curthread->td_frame) == NULL) + return (0); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + pc = tf->tf_sepc; + fp = tf->tf_s[0]; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + /* + * In an entry probe. The frame pointer has not yet been + * pushed (that happens in the function prologue). The + * best approach is to add the current pc as a missing top + * of stack and back the pc up to the caller, which is stored + * at the current stack pointer address since the call + * instruction puts it there right before the branch. + */ + pc = tf->tf_ra; + n++; + } + + n += dtrace_getustack_common(NULL, 0, pc, fp); return (0); } @@ -285,12 +315,36 @@ dtrace_getstackdepth(int aframes) } ulong_t -dtrace_getreg(struct trapframe *rp, uint_t reg) +dtrace_getreg(struct trapframe *frame, uint_t reg) { - - printf("IMPLEMENT ME: %s\n", __func__); - - return (0); + switch (reg) { + case REG_ZERO: + return (0); + case REG_RA: + return (frame->tf_ra); + case REG_SP: + return (frame->tf_sp); + case REG_GP: + return (frame->tf_gp); + case REG_TP: + return (frame->tf_tp); + case REG_T0 ... REG_T2: + return (frame->tf_t[reg - REG_T0]); + case REG_S0 ... REG_S1: + return (frame->tf_s[reg - REG_S0]); + case REG_A0 ... REG_A7: + return (frame->tf_a[reg - REG_A0]); + case REG_S2 ... REG_S11: + return (frame->tf_s[reg - REG_S2 + 2]); + case REG_T3 ... REG_T6: + return (frame->tf_t[reg - REG_T3 + 3]); + case REG_PC: + return (frame->tf_sepc); + default: + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + /* NOTREACHED */ } static int @@ -393,3 +447,68 @@ dtrace_fuword64(void *uaddr) return (dtrace_fuword64_nocheck(uaddr)); } + +int +dtrace_match_opcode(uint32_t insn, int match, int mask) +{ + if (((insn ^ match) & mask) == 0) + return (1); + + return (0); +} + +int +dtrace_instr_sdsp(uint32_t **instr) +{ + if (dtrace_match_opcode(**instr, (MATCH_SD | RS2_RA | RS1_SP), + (MASK_SD | RS2_MASK | RS1_MASK))) + return (1); + + return (0); +} + +int +dtrace_instr_c_sdsp(uint32_t **instr) +{ + uint16_t *instr1; + int i; + + for (i = 0; i < 2; i++) { + instr1 = (uint16_t *)(*instr) + i; + if (dtrace_match_opcode(*instr1, (MATCH_C_SDSP | RS2_C_RA), + (MASK_C_SDSP | RS2_C_MASK))) { + *instr = (uint32_t *)instr1; + return (1); + } + } + + return (0); +} + +int +dtrace_instr_ret(uint32_t **instr) +{ + if (dtrace_match_opcode(**instr, (MATCH_JALR | (X_RA << RS1_SHIFT)), + (MASK_JALR | RD_MASK | RS1_MASK | IMM_MASK))) + return (1); + + return (0); +} + +int +dtrace_instr_c_ret(uint32_t **instr) +{ + uint16_t *instr1; + int i; + + for (i = 0; i < 2; i++) { + instr1 = (uint16_t *)(*instr) + i; + if (dtrace_match_opcode(*instr1, + (MATCH_C_JR | (X_RA << RD_SHIFT)), (MASK_C_JR | RD_MASK))) { + *instr = (uint32_t *)instr1; + return (1); + } + } + + return (0); +} diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_subr.c b/sys/cddl/dev/dtrace/riscv/dtrace_subr.c index c1ac339b3a26..3a6aacd86fcd 100644 --- a/sys/cddl/dev/dtrace/riscv/dtrace_subr.c +++ b/sys/cddl/dev/dtrace/riscv/dtrace_subr.c @@ -21,26 +21,22 @@ * * Portions Copyright 2016-2018 Ruslan Bukin <br@bsdpad.com> * - * $FreeBSD$ - * */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/systm.h> -#include <sys/types.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/kmem.h> +#include <sys/proc.h> #include <sys/smp.h> #include <sys/dtrace_impl.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> #include <machine/vmparam.h> #include <machine/encoding.h> #include <machine/riscvreg.h> @@ -68,14 +64,18 @@ dtrace_invop_hdlr_t *dtrace_invop_hdlr; int dtrace_invop(uintptr_t addr, struct trapframe *frame) { + struct thread *td; dtrace_invop_hdlr_t *hdlr; int rval; + rval = 0; + td = curthread; + td->t_dtrace_trapframe = frame; for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) if ((rval = hdlr->dtih_func(addr, frame, 0)) != 0) - return (rval); - - return (0); + break; + td->t_dtrace_trapframe = NULL; + return (rval); } void @@ -162,7 +162,7 @@ dtrace_sync(void) * Returns nanoseconds since boot. */ uint64_t -dtrace_gethrtime() +dtrace_gethrtime(void) { struct timespec curtime; @@ -194,9 +194,7 @@ dtrace_trap(struct trapframe *frame, u_int type) * flag is cleared and finally re-scheduling is enabled. * * Check if DTrace has enabled 'no-fault' mode: - * */ - if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { /* * There are only a couple of trap types that are expected. @@ -206,15 +204,19 @@ dtrace_trap(struct trapframe *frame, u_int type) case SCAUSE_LOAD_ACCESS_FAULT: case SCAUSE_STORE_ACCESS_FAULT: case SCAUSE_INST_ACCESS_FAULT: + case SCAUSE_INST_PAGE_FAULT: + case SCAUSE_LOAD_PAGE_FAULT: + case SCAUSE_STORE_PAGE_FAULT: /* Flag a bad address. */ cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; - cpu_core[curcpu].cpuc_dtrace_illval = 0; + cpu_core[curcpu].cpuc_dtrace_illval = frame->tf_stval; /* * Offset the instruction pointer to the instruction * following the one causing the fault. */ - frame->tf_sepc += 4; + frame->tf_sepc += + dtrace_instr_size((uint8_t *)frame->tf_sepc); return (1); default: @@ -238,16 +240,6 @@ dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, } static int -match_opcode(uint32_t insn, int match, int mask) -{ - - if (((insn ^ match) & mask) == 0) - return (1); - - return (0); -} - -static int dtrace_invop_start(struct trapframe *frame) { register_t *sp; @@ -259,7 +251,7 @@ dtrace_invop_start(struct trapframe *frame) if (invop == 0) return (-1); - if (match_opcode(invop, (MATCH_SD | RS2_RA | RS1_SP), + if (dtrace_match_opcode(invop, (MATCH_SD | RS2_RA | RS1_SP), (MASK_SD | RS2_MASK | RS1_MASK))) { /* Non-compressed store of ra to sp */ imm = (invop >> 7) & 0x1f; @@ -270,14 +262,14 @@ dtrace_invop_start(struct trapframe *frame) return (0); } - if (match_opcode(invop, (MATCH_JALR | (X_RA << RS1_SHIFT)), + if (dtrace_match_opcode(invop, (MATCH_JALR | (X_RA << RS1_SHIFT)), (MASK_JALR | RD_MASK | RS1_MASK | IMM_MASK))) { /* Non-compressed ret */ frame->tf_sepc = frame->tf_ra; return (0); } - if (match_opcode(invop, (MATCH_C_SDSP | RS2_C_RA), + if (dtrace_match_opcode(invop, (MATCH_C_SDSP | RS2_C_RA), (MASK_C_SDSP | RS2_C_MASK))) { /* 'C'-compressed store of ra to sp */ uimm = ((invop >> 10) & 0x7) << 3; @@ -288,13 +280,16 @@ dtrace_invop_start(struct trapframe *frame) return (0); } - if (match_opcode(invop, (MATCH_C_JR | (X_RA << RD_SHIFT)), + if (dtrace_match_opcode(invop, (MATCH_C_JR | (X_RA << RD_SHIFT)), (MASK_C_JR | RD_MASK))) { /* 'C'-compressed ret */ frame->tf_sepc = frame->tf_ra; return (0); } + if (dtrace_match_opcode(invop, MATCH_C_NOP, MASK_C_NOP)) + return (0); + #ifdef INVARIANTS panic("Instruction %x doesn't match any opcode.", invop); #endif diff --git a/sys/cddl/dev/dtrace/riscv/instr_size.c b/sys/cddl/dev/dtrace/riscv/instr_size.c new file mode 100644 index 000000000000..bfdc962f4aa9 --- /dev/null +++ b/sys/cddl/dev/dtrace/riscv/instr_size.c @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright 2023 Christos Margiolis <christos@FreeBSD.org> + */ + +#include <sys/types.h> +#include <sys/dtrace.h> + +#include <machine/riscvreg.h> + +#define RVC_MASK 0x03 + +int +dtrace_instr_size(uint8_t *instr) +{ + /* Detect compressed instructions. */ + if ((~(*instr) & RVC_MASK) == 0) + return (INSN_SIZE); + else + return (INSN_C_SIZE); +} diff --git a/sys/cddl/dev/dtrace/riscv/regset.h b/sys/cddl/dev/dtrace/riscv/regset.h index f99b48f8354f..db9f2340b190 100644 --- a/sys/cddl/dev/dtrace/riscv/regset.h +++ b/sys/cddl/dev/dtrace/riscv/regset.h @@ -19,7 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -35,14 +34,46 @@ #define _REGSET_H /* - * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" */ #ifdef __cplusplus extern "C" { #endif -/* Place here */ +#define REG_ZERO 0 +#define REG_RA 1 +#define REG_SP 2 +#define REG_GP 3 +#define REG_TP 4 +#define REG_T0 5 +#define REG_T1 6 +#define REG_T2 7 +#define REG_S0 8 +#define REG_FP 8 +#define REG_S1 9 +#define REG_A0 10 +#define REG_A1 11 +#define REG_A2 12 +#define REG_A3 13 +#define REG_A4 14 +#define REG_A5 15 +#define REG_A6 16 +#define REG_A7 17 +#define REG_S2 18 +#define REG_S3 19 +#define REG_S4 20 +#define REG_S5 21 +#define REG_S6 22 +#define REG_S7 23 +#define REG_S8 24 +#define REG_S9 25 +#define REG_S10 26 +#define REG_S11 27 +#define REG_T3 28 +#define REG_T4 29 +#define REG_T5 30 +#define REG_T6 31 +#define REG_PC 32 #ifdef __cplusplus } diff --git a/sys/cddl/dev/dtrace/x86/dis_tables.c b/sys/cddl/dev/dtrace/x86/dis_tables.c index d130b3ac335e..1afadc94afc2 100644 --- a/sys/cddl/dev/dtrace/x86/dis_tables.c +++ b/sys/cddl/dev/dtrace/x86/dis_tables.c @@ -21,7 +21,8 @@ */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright 2016 Joyent, Inc. + * Copyright 2019 Joyent, Inc. + * Copyright 2020 Robert Mustacchi */ /* @@ -30,10 +31,9 @@ */ /* Copyright (c) 1988 AT&T */ -/* All Rights Reserved */ +/* All Rights Reserved */ /* - * $FreeBSD$ */ #include "dis_tables.h" @@ -48,8 +48,8 @@ * * The behavior of this file can be controlled by one of the following flags: * - * DIS_TEXT Include text for disassembly - * DIS_MEM Include memory-size calculations + * DIS_TEXT Include text for disassembly + * DIS_MEM Include memory-size calculations * * Either or both of these can be defined. * @@ -69,7 +69,7 @@ extern size_t strlcat(char *, const char *, size_t); #endif -#define TERM 0 /* used to indicate that the 'indirect' */ +#define TERM 0 /* used to indicate that the 'indirect' */ /* field terminates - no pointer. */ /* Used to decode instructions. */ @@ -88,7 +88,8 @@ typedef struct instable { uint_t it_invalid32:1; /* invalid in IA32 */ uint_t it_stackop:1; /* push/pop stack operation */ uint_t it_vexwoxmm:1; /* VEX instructions that don't use XMM/YMM */ - uint_t it_avxsuf:1; /* AVX suffix required */ + uint_t it_avxsuf:2; /* AVX2/AVX512 suffix rqd. */ + uint_t it_vexopmask:1; /* VEX inst. that use opmask */ } instable_t; /* @@ -118,7 +119,7 @@ enum { SEG, MR, RM, - RM_66r, /* RM, but with a required 0x66 prefix */ + RM_66r, /* RM, but with a required 0x66 prefix */ IA, MA, SD, @@ -137,7 +138,7 @@ enum { NORM, /* instructions w/o ModR/M byte, no memory access */ IMPLMEM, /* instructions w/o ModR/M byte, implicit mem access */ O, /* for call */ - JTAB, /* jump table */ + JTAB, /* jump table */ IMUL, /* for 186 iimul instr */ CBW, /* so data16 can be evaluated for cbw and variants */ MvI, /* for 186 logicals */ @@ -172,7 +173,7 @@ enum { MMO, /* Prefixable MMX/SIMD-Int mm/mem -> mm */ MMOIMPL, /* Prefixable MMX/SIMD-Int mm -> mm (mem) */ MMO3P, /* Prefixable MMX/SIMD-Int mm -> r32,imm8 */ - MMOM3, /* Prefixable MMX/SIMD-Int mm -> r32 */ + MMOM3, /* Prefixable MMX/SIMD-Int mm -> r32 */ MMOS, /* Prefixable MMX/SIMD-Int mm -> mm/mem */ MMOMS, /* Prefixable MMX/SIMD-Int mm -> mem */ MMOPM, /* MMX/SIMD-Int mm/mem -> mm,imm8 */ @@ -190,33 +191,34 @@ enum { XMMOXMM, /* Prefixable SIMD xmm/mem -> mm */ XMMOM, /* Prefixable SIMD xmm -> mem */ XMMOMS, /* Prefixable SIMD mem -> xmm */ - XMM, /* SIMD xmm/mem -> xmm */ + XMM, /* SIMD xmm/mem -> xmm */ XMM_66r, /* SIMD 0x66 prefix required xmm/mem -> xmm */ - XMM_66o, /* SIMD 0x66 prefix optional xmm/mem -> xmm */ + XMM_66o, /* SIMD 0x66 prefix optional xmm/mem -> xmm */ XMMXIMPL, /* SIMD xmm -> xmm (mem) */ XMM3P, /* SIMD xmm -> r32,imm8 */ XMM3PM_66r, /* SIMD 0x66 prefix required xmm -> r32/mem,imm8 */ - XMMP, /* SIMD xmm/mem w/to xmm,imm8 */ + XMMP, /* SIMD xmm/mem w/to xmm,imm8 */ XMMP_66o, /* SIMD 0x66 prefix optional xmm/mem w/to xmm,imm8 */ XMMP_66r, /* SIMD 0x66 prefix required xmm/mem w/to xmm,imm8 */ - XMMPRM, /* SIMD r32/mem -> xmm,imm8 */ + XMMPRM, /* SIMD r32/mem -> xmm,imm8 */ XMMPRM_66r, /* SIMD 0x66 prefix required r32/mem -> xmm,imm8 */ XMMS, /* SIMD xmm -> xmm/mem */ - XMMM, /* SIMD mem -> xmm */ + XMMM, /* SIMD mem -> xmm */ XMMM_66r, /* SIMD 0x66 prefix required mem -> xmm */ XMMMS, /* SIMD xmm -> mem */ - XMM3MX, /* SIMD r32/mem -> xmm */ - XMM3MXS, /* SIMD xmm -> r32/mem */ - XMMSH, /* SIMD xmm,imm8 */ - XMMXM3, /* SIMD xmm/mem -> r32 */ - XMMX3, /* SIMD xmm -> r32 */ - XMMXMM, /* SIMD xmm/mem -> mm */ - XMMMX, /* SIMD mm -> xmm */ - XMMXM, /* SIMD xmm -> mm */ - XMMX2I, /* SIMD xmm -> xmm, imm, imm */ - XMM2I, /* SIMD xmm, imm, imm */ + XMM3MX, /* SIMD r32/mem -> xmm */ + XMM3MXS, /* SIMD xmm -> r32/mem */ + XMMSH, /* SIMD xmm,imm8 */ + XMMXM3, /* SIMD xmm/mem -> r32 */ + XMMX3, /* SIMD xmm -> r32 */ + XMMXMM, /* SIMD xmm/mem -> mm */ + XMMMX, /* SIMD mm -> xmm */ + XMMXM, /* SIMD xmm -> mm */ + XMMX2I, /* SIMD xmm -> xmm, imm, imm */ + XMM2I, /* SIMD xmm, imm, imm */ XMMFENCE, /* SIMD lfence or mfence */ XMMSFNC, /* SIMD sfence (none or mem) */ + FSGS, /* FSGSBASE if reg */ XGETBV_XSETBV, VEX_NONE, /* VEX no operand */ VEX_MO, /* VEX mod_rm -> implicit reg */ @@ -224,25 +226,32 @@ enum { VEX_VRMrX, /* VEX mod_rm, VEX.vvvv -> mod_rm */ VEX_RRX, /* VEX VEX.vvvv, mod_reg -> mod_rm */ VEX_RMRX, /* VEX VEX.vvvv, mod_rm, imm8[7:4] -> mod_reg */ - VEX_MX, /* VEX mod_rm -> mod_reg */ - VEX_MXI, /* VEX mod_rm, imm8 -> mod_reg */ - VEX_XXI, /* VEX mod_rm, imm8 -> VEX.vvvv */ - VEX_MR, /* VEX mod_rm -> mod_reg */ - VEX_RRI, /* VEX mod_reg, mod_rm -> implicit(eflags/r32) */ - VEX_RX, /* VEX mod_reg -> mod_rm */ - VEX_RR, /* VEX mod_rm -> mod_reg */ - VEX_RRi, /* VEX mod_rm, imm8 -> mod_reg */ - VEX_RM, /* VEX mod_reg -> mod_rm */ + VEX_MX, /* VEX mod_rm -> mod_reg */ + VEX_MXI, /* VEX mod_rm, imm8 -> mod_reg */ + VEX_XXI, /* VEX mod_rm, imm8 -> VEX.vvvv */ + VEX_MR, /* VEX mod_rm -> mod_reg */ + VEX_RRI, /* VEX mod_reg, mod_rm -> implicit(eflags/r32) */ + VEX_RX, /* VEX mod_reg -> mod_rm */ + VEX_KRR, /* VEX mod_rm -> mod_reg */ + VEX_KMR, /* VEX mod_reg -> mod_rm */ + VEX_KRM, /* VEX mod_rm -> mod_reg */ + VEX_RR, /* VEX mod_rm -> mod_reg */ + VEX_RRi, /* VEX mod_rm, imm8 -> mod_reg */ + VEX_RM, /* VEX mod_reg -> mod_rm */ VEX_RIM, /* VEX mod_reg, imm8 -> mod_rm */ - VEX_RRM, /* VEX VEX.vvvv, mod_reg -> mod_rm */ - VEX_RMX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_RRM, /* VEX VEX.vvvv, mod_reg -> mod_rm */ + VEX_RMX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ VEX_SbVM, /* VEX SIB, VEX.vvvv -> mod_rm */ VMx, /* vmcall/vmlaunch/vmresume/vmxoff */ VMxo, /* VMx instruction with optional prefix */ SVM, /* AMD SVM instructions */ BLS, /* BLSR, BLSMSK, BLSI */ FMA, /* FMA instructions, all VEX_RMrX */ - ADX /* ADX instructions, support REX.w, mod_rm->mod_reg */ + ADX, /* ADX instructions, support REX.w, mod_rm->mod_reg */ + EVEX_RX, /* EVEX mod_reg -> mod_rm */ + EVEX_MX, /* EVEX mod_rm -> mod_reg */ + EVEX_RMrX, /* EVEX EVEX.vvvv, mod_rm -> mod_reg */ + EVEX_RMRX /* EVEX EVEX.vvvv, mod_rm, imm8 -> mod_reg */ }; /* @@ -280,7 +289,9 @@ enum { * IND - indirect to another to another table * "T" - means to Terminate indirections (this is the final opcode) * "S" - means "operand length suffix required" - * "Sa" - means AVX2 suffix (d/q) required + * "Sa" - means AVX2 suffix (q/d) required + * "Sq" - means AVX512 suffix (q/d) required + * "Sd" - means AVX512 suffix (d/s) required * "NS" - means "no suffix" which is the operand length suffix of the opcode * "Z" - means instruction size arg required * "u" - means the opcode is invalid in IA32 but valid in amd64 @@ -288,8 +299,13 @@ enum { * "y" - means the operand size is always 64 bits in 64 bit mode * "p" - means push/pop stack operation * "vr" - means VEX instruction that operates on normal registers, not fpu + * "vo" - means VEX instruction that operates on opmask registers, not fpu */ +#define AVS2 (uint_t)1 /* it_avxsuf: AVX2 q/d suffix handling */ +#define AVS5Q (uint_t)2 /* it_avxsuf: AVX512 q/d suffix handling */ +#define AVS5D (uint_t)3 /* it_avxsuf: AVX512 d/s suffix handling */ + #if defined(DIS_TEXT) && defined(DIS_MEM) #define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0} #define INDx(table) {(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0} @@ -301,12 +317,15 @@ enum { #define TNSZ(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0} #define TNSZy(name, amode, sz) {TERM, amode, name, 0, sz, 0, 1, 0, 0} #define TNSZvr(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0, 1} +#define TSvo(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 0, 0, 1} #define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0} #define TSx(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0, 0} #define TSy(name, amode) {TERM, amode, name, 1, 0, 0, 1, 0, 0} #define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 1} #define TSZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0} -#define TSaZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, 1} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, AVS2} +#define TSq(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0, 0, AVS5Q} +#define TSd(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0, 0, AVS5D} #define TSZx(name, amode, sz) {TERM, amode, name, 1, sz, 1, 0, 0, 0} #define TSZy(name, amode, sz) {TERM, amode, name, 1, sz, 0, 1, 0, 0} #define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} @@ -321,12 +340,14 @@ enum { #define TNSZ(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0} #define TNSZy(name, amode, sz) {TERM, amode, name, 0, 0, 1, 0, 0} #define TNSZvr(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0, 1} +#define TSvo(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 0, 1} #define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0} #define TSx(name, amode) {TERM, amode, name, 1, 1, 0, 0, 0} #define TSy(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0} #define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 1} #define TSZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0} -#define TSaZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 1} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0, 0, AVS2} +#define TSq(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0, AVS5Q} #define TSZx(name, amode, sz) {TERM, amode, name, 1, 1, 0, 0, 0} #define TSZy(name, amode, sz) {TERM, amode, name, 1, 0, 1, 0, 0} #define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} @@ -341,12 +362,14 @@ enum { #define TNSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} #define TNSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} #define TNSZvr(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 1} +#define TSvo(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, 0, 1} #define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} #define TSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} #define TSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} #define TSp(name, amode) {TERM, amode, 0, 0, 0, 0, 1} #define TSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} -#define TSaZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 1} +#define TSaZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, AVS2} +#define TSq(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, AVS5Q} #define TSZx(name, amode, sz) {TERM, amode, sz, 1, 0, 0, 0} #define TSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} #define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0} @@ -361,12 +384,15 @@ enum { #define TNSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} #define TNSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} #define TNSZvr(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 1} +#define TSvo(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, 1} #define TS(name, amode) {TERM, amode, 0, 0, 0, 0} #define TSx(name, amode) {TERM, amode, 1, 0, 0, 0} #define TSy(name, amode) {TERM, amode, 0, 1, 0, 0} #define TSp(name, amode) {TERM, amode, 0, 0, 0, 1} #define TSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} -#define TSaZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 1} +#define TSaZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, AVS2} +#define TSq(name, amode) {TERM, amode, 0, 0, 0, 0, 0, AVS5Q} +#define TSd(name, amode) {TERM, amode, 0, 0, 0, 0, 0, AVS5D} #define TSZx(name, amode, sz) {TERM, amode, 1, 0, 0, 0} #define TSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} #define INVALID {TERM, UNKNOWN, 0, 0, 0, 0} @@ -470,14 +496,41 @@ const char *const dis_MMREG[16] = { "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7" }; -const char *const dis_XMMREG[16] = { - "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", - "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15" +const char *const dis_XMMREG[32] = { + "%xmm0", "%xmm1", "%xmm2", "%xmm3", + "%xmm4", "%xmm5", "%xmm6", "%xmm7", + "%xmm8", "%xmm9", "%xmm10", "%xmm11", + "%xmm12", "%xmm13", "%xmm14", "%xmm15", + "%xmm16", "%xmm17", "%xmm18", "%xmm19", + "%xmm20", "%xmm21", "%xmm22", "%xmm23", + "%xmm24", "%xmm25", "%xmm26", "%xmm27", + "%xmm28", "%xmm29", "%xmm30", "%xmm31", +}; + +const char *const dis_YMMREG[32] = { + "%ymm0", "%ymm1", "%ymm2", "%ymm3", + "%ymm4", "%ymm5", "%ymm6", "%ymm7", + "%ymm8", "%ymm9", "%ymm10", "%ymm11", + "%ymm12", "%ymm13", "%ymm14", "%ymm15", + "%ymm16", "%ymm17", "%ymm18", "%ymm19", + "%ymm20", "%ymm21", "%ymm22", "%ymm23", + "%ymm24", "%ymm25", "%ymm26", "%ymm27", + "%ymm28", "%ymm29", "%ymm30", "%ymm31", }; -const char *const dis_YMMREG[16] = { - "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", - "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15" +const char *const dis_ZMMREG[32] = { + "%zmm0", "%zmm1", "%zmm2", "%zmm3", + "%zmm4", "%zmm5", "%zmm6", "%zmm7", + "%zmm8", "%zmm9", "%zmm10", "%zmm11", + "%zmm12", "%zmm13", "%zmm14", "%zmm15", + "%zmm16", "%zmm17", "%zmm18", "%zmm19", + "%zmm20", "%zmm21", "%zmm22", "%zmm23", + "%zmm24", "%zmm25", "%zmm26", "%zmm27", + "%zmm28", "%zmm29", "%zmm30", "%zmm31", +}; + +const char *const dis_KOPMASKREG[8] = { + "%k0", "%k1", "%k2", "%k3", "%k4", "%k5", "%k6", "%k7" }; const char *const dis_SEGREG[16] = { @@ -512,11 +565,16 @@ const instable_t dis_opMOVSLD = TNS("movslq",MOVSXZ); const instable_t dis_opPause = TNS("pause", NORM); /* + * "decode table" for wbnoinvd instruction + */ +const instable_t dis_opWbnoinvd = TNS("wbnoinvd", NORM); + +/* * Decode table for 0x0F00 opcodes */ const instable_t dis_op0F00[8] = { -/* [0] */ TNS("sldt",M), TNS("str",M), TNSy("lldt",M), TNSy("ltr",M), +/* [0] */ TNS("sldt",M), TNS("str",M), TNSy("lldt",M), TNSy("ltr",M), /* [4] */ TNSZ("verr",M,2), TNSZ("verw",M,2), INVALID, INVALID, }; @@ -527,7 +585,7 @@ const instable_t dis_op0F00[8] = { const instable_t dis_op0F01[8] = { /* [0] */ TNSZ("sgdt",VMx,6), TNSZ("sidt",MONITOR_MWAIT,6), TNSZ("lgdt",XGETBV_XSETBV,6), TNSZ("lidt",SVM,6), -/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS_RDTSCP), +/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS_RDTSCP), }; /* @@ -540,14 +598,22 @@ const instable_t dis_op0F18[8] = { }; /* - * Decode table for 0x0FAE opcodes -- SIMD state save/restore + * Decode table for 0x0FAE opcodes -- SIMD state save/restore */ const instable_t dis_op0FAE[8] = { -/* [0] */ TNSZ("fxsave",M,512), TNSZ("fxrstor",M,512), TNS("ldmxcsr",M), TNS("stmxcsr",M), +/* [0] */ TNSZ("fxsave",FSGS,512),TNSZ("fxrstor",FSGS,512),TNS("ldmxcsr",FSGS), TNS("stmxcsr",FSGS), /* [4] */ TNSZ("xsave",M,512), TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), }; /* + * Decode table for 0xF30FAE opcodes -- FSGSBASE + */ +const instable_t dis_opF30FAE[8] = { +/* [0] */ TNSx("rdfsbase",FSGS), TNSx("rdgsbase",FSGS), TNSx("wrfsbase",FSGS), TNSx("wrgsbase",FSGS), +/* [4] */ INVALID, INVALID, INVALID, INVALID, +}; + +/* * Decode table for 0x0FBA opcodes */ @@ -558,17 +624,17 @@ const instable_t dis_op0FBA[8] = { }; /* - * Decode table for 0x0FC7 opcode (group 9) + * Decode table for 0x0FC7 opcode (group 9) */ const instable_t dis_op0FC7[8] = { -/* [0] */ INVALID, TNS("cmpxchg8b",M), INVALID, INVALID, -/* [4] */ INVALID, INVALID, TNS("vmptrld",MG9), TNS("vmptrst",MG9), +/* [0] */ INVALID, TNS("cmpxchg8b",M), INVALID, TNS("xrstors",MG9), +/* [4] */ TNS("xsavec",MG9), TNS("xsaves",MG9), TNS("vmptrld",MG9), TNS("vmptrst",MG9), }; /* - * Decode table for 0x0FC7 opcode (group 9) mode 3 + * Decode table for 0x0FC7 opcode (group 9) mode 3 */ const instable_t dis_op0FC7m3[8] = { @@ -578,7 +644,7 @@ const instable_t dis_op0FC7m3[8] = { }; /* - * Decode table for 0x0FC7 opcode with 0x66 prefix + * Decode table for 0x0FC7 opcode with 0x66 prefix */ const instable_t dis_op660FC7[8] = { @@ -588,7 +654,7 @@ const instable_t dis_op660FC7[8] = { }; /* - * Decode table for 0x0FC7 opcode with 0xF3 prefix + * Decode table for 0x0FC7 opcode with 0xF3 prefix */ const instable_t dis_opF30FC7[8] = { @@ -621,7 +687,7 @@ const instable_t dis_op0F7123[4][8] = { /* .4 */ TNS("psrad",MMOSH), INVALID, TNS("pslld",MMOSH), INVALID, }, { /* [73].0 */ INVALID, INVALID, TNS("psrlq",MMOSH), TNS("INVALID",MMOSH), -/* .4 */ INVALID, INVALID, TNS("psllq",MMOSH), TNS("INVALID",MMOSH), +/* .4 */ INVALID, INVALID, TNS("psllq",MMOSH), TNS("INVALID",MMOSH), } }; /* @@ -757,9 +823,9 @@ const instable_t dis_opAVX660F[256] = { /* [38] */ INVALID, INVALID, INVALID, INVALID, /* [3C] */ INVALID, INVALID, INVALID, INVALID, -/* [40] */ INVALID, INVALID, INVALID, INVALID, -/* [44] */ INVALID, INVALID, INVALID, INVALID, -/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [40] */ INVALID, TSvo("kand",VEX_RMX), TSvo("kandn",VEX_RMX), INVALID, +/* [44] */ TSvo("knot",VEX_MX), TSvo("kor",VEX_RMX), TSvo("kxnor",VEX_RMX), TSvo("kxor",VEX_RMX), +/* [48] */ INVALID, INVALID, TSvo("kadd",VEX_RMX), TSvo("kunpck",VEX_RMX), /* [4C] */ INVALID, INVALID, INVALID, INVALID, /* [50] */ TNS("vmovmskpd",VEX_MR), TNSZ("vsqrtpd",VEX_MX,16), INVALID, INVALID, @@ -782,9 +848,9 @@ const instable_t dis_opAVX660F[256] = { /* [88] */ INVALID, INVALID, INVALID, INVALID, /* [8C] */ INVALID, INVALID, INVALID, INVALID, -/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [90] */ TSvo("kmov",VEX_KRM), TSvo("kmov",VEX_KMR), TSvo("kmov",VEX_KRR), TSvo("kmov",VEX_MR), /* [94] */ INVALID, INVALID, INVALID, INVALID, -/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ TSvo("kortest",VEX_MX), TSvo("ktest",VEX_MX), INVALID, INVALID, /* [9C] */ INVALID, INVALID, INVALID, INVALID, /* [A0] */ INVALID, INVALID, INVALID, INVALID, @@ -949,7 +1015,7 @@ const instable_t dis_opAVXF20F[256] = { /* [88] */ INVALID, INVALID, INVALID, INVALID, /* [0C] */ INVALID, INVALID, INVALID, INVALID, -/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [90] */ INVALID, INVALID, TSvo("kmov",VEX_KRR), TSvo("kmov",VEX_MR), /* [94] */ INVALID, INVALID, INVALID, INVALID, /* [98] */ INVALID, INVALID, INVALID, INVALID, /* [9C] */ INVALID, INVALID, INVALID, INVALID, @@ -1396,6 +1462,505 @@ const instable_t dis_opAVXF30F[256] = { /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; + +/* + * Table for instructions with an EVEX prefix followed by 0F. + */ +const instable_t dis_opEVEX0F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNS("vmovups",EVEX_MX), TNS("vmovups",EVEX_RX), INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ TNS("vmovaps",EVEX_MX), TNS("vmovaps",EVEX_RX), INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ TNS("vandps",EVEX_RMrX),TNS("vandnps",EVEX_RMrX),TNS("vorps",EVEX_RMrX),TNS("vxorps",EVEX_RMrX), +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +/* + * Decode tables for EVEX 66 0F + */ +const instable_t dis_opEVEX660F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ TNS("vmovupd",EVEX_MX), TNS("vmovupd",EVEX_RX), INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ TNS("vmovapd",EVEX_MX), TNS("vmovapd",EVEX_RX), INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ TNS("vandpd",EVEX_RMrX),TNS("vandnpd",EVEX_RMrX),TNS("vorpd",EVEX_RMrX),TNS("vxorpd",EVEX_RMrX), +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, TNS("vmovdqa",EVEX_MX), + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, TNS("vmovdqa",EVEX_RX), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, TSq("vpand",EVEX_RMrX), +/* [DC] */ INVALID, INVALID, INVALID, TSq("vpandn",EVEX_RMrX), + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, TSq("vpor",EVEX_RMrX), +/* [EC] */ INVALID, INVALID, INVALID, TSq("vpxor",EVEX_RMrX), + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opEVEX660F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ TNSZ("vpdpbusd",EVEX_RMrX,16),TNSZ("vpdpbusds",EVEX_RMrX,16),TNSZ("vpdpwssd",EVEX_RMrX,16),TNSZ("vpdpwssds",EVEX_RMrX,16), +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, TNS("vgf2p8mulb",EVEX_RMrX), + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ TNSZ("vaesenc",EVEX_RMrX,16),TNSZ("vaesenclast",EVEX_RMrX,16),TNSZ("vaesdec",EVEX_RMrX,16),TNSZ("vaesdeclast",EVEX_RMrX,16), + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opEVEX660F3A[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ TNSZ("vpclmulqdq",EVEX_RMRX,16),INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, TNS("vgf2p8affineqb",EVEX_RMRX),TNS("vgf2p8affineinvqb",EVEX_RMRX), + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + + +const instable_t dis_opEVEXF20F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, TNS("vmovdqu",EVEX_MX), + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, TNS("vmovdqu",EVEX_RX), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opEVEXF30F[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, TNS("vmovdqu",EVEX_MX), + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, TNS("vmovdqu",EVEX_RX), + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; /* * The following two tables are used to encode crc32 and movbe * since they share the same opcodes. @@ -1483,7 +2048,7 @@ const instable_t dis_op0F38[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, /* [C8] */ TNSZ("sha1nexte",XMM,16),TNSZ("sha1msg1",XMM,16),TNSZ("sha1msg2",XMM,16),TNSZ("sha256rnds2",XMM,16), -/* [CC] */ TNSZ("sha256msg1",XMM,16),TNSZ("sha256msg2",XMM,16),INVALID, INVALID, +/* [CC] */ TNSZ("sha256msg1",XMM,16),TNSZ("sha256msg2",XMM,16),INVALID, TNS("gf2p8mulb",XMM_66r), /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, @@ -1564,7 +2129,7 @@ const instable_t dis_opAVX660F38[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, /* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, TNS("vgf2p8mulb",VEX_RMrX), /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, @@ -1645,7 +2210,7 @@ const instable_t dis_op0F3A[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, /* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ TNSZ("sha1rnds4",XMMP,16),INVALID, INVALID, INVALID, +/* [CC] */ TNSZ("sha1rnds4",XMMP,16),INVALID, TNS("gf2p8affineqb",XMMP_66r),TNS("gf2p8affineinvqb",XMMP_66r), /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, @@ -1679,7 +2244,7 @@ const instable_t dis_opAVX660F3A[256] = { /* [28] */ INVALID, INVALID, INVALID, INVALID, /* [2C] */ INVALID, INVALID, INVALID, INVALID, -/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [30] */ TSvo("kshiftr",VEX_MXI), TSvo("kshiftr",VEX_MXI), TSvo("kshiftl",VEX_MXI), TSvo("kshiftl",VEX_MXI), /* [34] */ INVALID, INVALID, INVALID, INVALID, /* [38] */ TNSZ("vinserti128",VEX_RMRX,16),TNSZ("vextracti128",VEX_RIM,16),INVALID, INVALID, /* [3C] */ INVALID, INVALID, INVALID, INVALID, @@ -1727,7 +2292,7 @@ const instable_t dis_opAVX660F3A[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, /* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, TNS("vgf2p8affineqb",VEX_RMRX),TNS("vgf2p8affineinvqb",VEX_RMRX), /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, @@ -1746,8 +2311,8 @@ const instable_t dis_opAVX660F3A[256] = { }; /* - * Decode table for 0x0F0D which uses the first byte of the mod_rm to - * indicate a sub-code. + * Decode table for 0x0F0D which uses the first byte of the mod_rm to + * indicate a sub-code. */ const instable_t dis_op0F0D[8] = { /* [00] */ INVALID, TNS("prefetchw",PREF), TNS("prefetchwt1",PREF),INVALID, @@ -1776,7 +2341,7 @@ const instable_t dis_op0F[16][16] = { /* [2C] */ TNSZ("cvttps2pi",XMMOXMM,8),TNSZ("cvtps2pi",XMMOXMM,8),TNSZ("ucomiss",XMMO,4),TNSZ("comiss",XMMO,4), }, { /* [30] */ TNS("wrmsr",NORM), TNS("rdtsc",NORM), TNS("rdmsr",NORM), TNS("rdpmc",NORM), -/* [34] */ TNSx("sysenter",NORM), TNSx("sysexit",NORM), INVALID, INVALID, +/* [34] */ TNS("sysenter",NORM), TNS("sysexit",NORM), INVALID, INVALID, /* [38] */ INVALID, INVALID, INVALID, INVALID, /* [3C] */ INVALID, INVALID, INVALID, INVALID, }, { @@ -1821,7 +2386,7 @@ const instable_t dis_op0F[16][16] = { /* [BC] */ TS("bsf",MRw), TS("bsr",MRw), TS("movsb",MOVZ), TNS("movswl",MOVZ), }, { /* [C0] */ TNS("xaddb",XADDB), TS("xadd",RMw), TNSZ("cmpps",XMMOPM,16),TNS("movnti",RM), -/* [C4] */ TNSZ("pinsrw",MMOPRM,2),TNS("pextrw",MMO3P), TNSZ("shufps",XMMOPM,16),IND(dis_op0FC7), +/* [C4] */ TNSZ("pinsrw",MMOPRM,2),TNS("pextrw",MMO3P), TNSZ("shufps",XMMOPM,16),IND(dis_op0FC7), /* [C8] */ INVALID, INVALID, INVALID, INVALID, /* [CC] */ INVALID, INVALID, INVALID, INVALID, }, { @@ -1863,9 +2428,9 @@ const instable_t dis_opAVX0F[16][16] = { /* [38] */ INVALID, INVALID, INVALID, INVALID, /* [3C] */ INVALID, INVALID, INVALID, INVALID, }, { -/* [40] */ INVALID, INVALID, INVALID, INVALID, -/* [44] */ INVALID, INVALID, INVALID, INVALID, -/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [40] */ INVALID, TSvo("kand",VEX_RMX), TSvo("kandn",VEX_RMX), INVALID, +/* [44] */ TSvo("knot",VEX_MX), TSvo("kor",VEX_RMX), TSvo("kxnor",VEX_RMX), TSvo("kxor",VEX_RMX), +/* [48] */ INVALID, INVALID, TSvo("kadd",VEX_RMX), TSvo("kunpck",VEX_RMX), /* [4C] */ INVALID, INVALID, INVALID, INVALID, }, { /* [50] */ TNS("vmovmskps",VEX_MR), TNSZ("vsqrtps",VEX_MX,16), TNSZ("vrsqrtps",VEX_MX,16),TNSZ("vrcpps",VEX_MX,16), @@ -1888,9 +2453,9 @@ const instable_t dis_opAVX0F[16][16] = { /* [88] */ INVALID, INVALID, INVALID, INVALID, /* [8C] */ INVALID, INVALID, INVALID, INVALID, }, { -/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [90] */ TSvo("kmov",VEX_KRM), TSvo("kmov",VEX_KMR), TSvo("kmov",VEX_KRR), TSvo("kmov",VEX_MR), /* [94] */ INVALID, INVALID, INVALID, INVALID, -/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ TSvo("kortest",VEX_MX), TSvo("ktest",VEX_MX), INVALID, INVALID, /* [9C] */ INVALID, INVALID, INVALID, INVALID, }, { /* [A0] */ INVALID, INVALID, INVALID, INVALID, @@ -1904,7 +2469,7 @@ const instable_t dis_opAVX0F[16][16] = { /* [BC] */ INVALID, INVALID, INVALID, INVALID, }, { /* [C0] */ INVALID, INVALID, TNSZ("vcmpps",VEX_RMRX,16),INVALID, -/* [C4] */ INVALID, INVALID, TNSZ("vshufps",VEX_RMRX,16),INVALID, +/* [C4] */ INVALID, INVALID, TNSZ("vshufps",VEX_RMRX,16),INVALID, /* [C8] */ INVALID, INVALID, INVALID, INVALID, /* [CC] */ INVALID, INVALID, INVALID, INVALID, }, { @@ -2200,7 +2765,7 @@ const instable_t dis_distable[16][16] = { /* [5,8] */ TSp("pop",R), TSp("pop",R), TSp("pop",R), TSp("pop",R), /* [5,C] */ TSp("pop",R), TSp("pop",R), TSp("pop",R), TSp("pop",R), }, { -/* [6,0] */ TSZx("pusha",IMPLMEM,28),TSZx("popa",IMPLMEM,28), TSx("bound",MR), TNS("arpl",RMw), +/* [6,0] */ TSZx("pusha",IMPLMEM,28),TSZx("popa",IMPLMEM,28), TSx("bound",RM), TNS("arpl",RMw), /* [6,4] */ TNS("%fs:",OVERRIDE), TNS("%gs:",OVERRIDE), TNS("data16",DM), TNS("addr16",AM), /* [6,8] */ TSp("push",I), TS("imul",IMUL), TSp("push",Ib), TS("imul",IMUL), /* [6,C] */ TNSZ("insb",IMPLMEM,1), TSZ("ins",IMPLMEM,4), TNSZ("outsb",IMPLMEM,1),TSZ("outs",IMPLMEM,4), @@ -2230,7 +2795,7 @@ const instable_t dis_distable[16][16] = { /* [B,8] */ TS("mov",IR), TS("mov",IR), TS("mov",IR), TS("mov",IR), /* [B,C] */ TS("mov",IR), TS("mov",IR), TS("mov",IR), TS("mov",IR), }, { -/* [C,0] */ IND(dis_opC0), IND(dis_opC1), TNSyp("ret",RET), TNSyp("ret",NORM), +/* [C,0] */ IND(dis_opC0), IND(dis_opC1), TNSyp("ret",RET), TNSyp("ret",NORM), /* [C,4] */ TNSx("les",MR), TNSx("lds",MR), TNS("movb",IMw), TS("mov",IMw), /* [C,8] */ TNSyp("enter",ENTER), TNSyp("leave",NORM), TNS("lret",RET), TNS("lret",NORM), /* [C,C] */ TNS("int",INT3), TNS("int",INTx), TNSx("into",NORM), TNS("iret",NORM), @@ -2275,16 +2840,25 @@ const instable_t dis_distable[16][16] = { #define REX_B 0x01 /* extends ModRM r_m, SIB base, or opcode reg */ /* - * These are the individual fields of a VEX prefix. + * These are the individual fields of a VEX/EVEX prefix. */ #define VEX_R 0x08 /* REX.R in 1's complement form */ #define VEX_X 0x04 /* REX.X in 1's complement form */ #define VEX_B 0x02 /* REX.B in 1's complement form */ + +/* Additional EVEX prefix definitions */ +#define EVEX_R 0x01 /* REX.R' in 1's complement form */ +#define EVEX_OPREG_MASK 0x7 /* bit mask for selecting opmask register number */ +#define EVEX_ZERO_MASK 0x80 /* bit mask for selecting zeroing */ + /* Vector Length, 0: scalar or 128-bit vector, 1: 256-bit vector */ #define VEX_L 0x04 +/* Vector Length, 0: scalar or 128-bit vector, 1: 256-bit vector, 2: 512-bit */ +#define EVEX_L 0x06 /* bit mask for EVEX.L'L vector length/RC */ #define VEX_W 0x08 /* opcode specific, use like REX.W */ #define VEX_m 0x1F /* VEX m-mmmm field */ -#define VEX_v 0x78 /* VEX register specifier */ +#define EVEX_m 0x3 /* EVEX mm field */ +#define VEX_v 0x78 /* VEX/EVEX register specifier */ #define VEX_p 0x03 /* VEX pp field, opcode extension */ /* VEX m-mmmm field, only used by three bytes prefix */ @@ -2324,6 +2898,8 @@ static int isize64[] = {1, 2, 4, 8}; #define TEST_OPND 7 /* "value" used to indicate a test reg */ #define WORD_OPND 8 /* w-bit value indicating word size reg */ #define YMM_OPND 9 /* "value" used to indicate a ymm reg */ +#define KOPMASK_OPND 10 /* "value" used to indicate an opmask reg */ +#define ZMM_OPND 11 /* "value" used to indicate a zmm reg */ /* * The AVX2 gather instructions are a bit of a mess. While there's a pattern, @@ -2338,7 +2914,7 @@ static int isize64[] = {1, 2, 4, 8}; * * We further have to subdivide this based on the value of VEX_W and the value * of VEX_L. The array is constructed to be indexed as: - * [opcode - 0x90][VEX_W][VEX_L]. + * [opcode - 0x90][VEX_W][VEX_L]. */ /* w = 0, 0x90 */ typedef struct dis_gather_regs { @@ -2501,6 +3077,157 @@ dtrace_vex_adjust(uint_t vex_byte1, uint_t mode, uint_t *reg, uint_t *r_m) } /* + * Adjust the instruction mnemonic with the appropriate suffix. + */ +/* ARGSUSED */ +static void +dtrace_evex_mnem_adjust(dis86_t *x, const instable_t *dp, uint_t vex_W, + uint_t evex_byte2) +{ +#ifdef DIS_TEXT + if (dp == &dis_opEVEX660F[0x7f] || /* vmovdqa */ + dp == &dis_opEVEX660F[0x6f]) { + (void) strlcat(x->d86_mnem, vex_W != 0 ? "64" : "32", + OPLEN); + } + + if (dp == &dis_opEVEXF20F[0x7f] || /* vmovdqu */ + dp == &dis_opEVEXF20F[0x6f] || + dp == &dis_opEVEXF30F[0x7f] || + dp == &dis_opEVEXF30F[0x6f]) { + switch (evex_byte2 & 0x81) { + case 0x0: + (void) strlcat(x->d86_mnem, "32", OPLEN); + break; + case 0x1: + (void) strlcat(x->d86_mnem, "8", OPLEN); + break; + case 0x80: + (void) strlcat(x->d86_mnem, "64", OPLEN); + break; + case 0x81: + (void) strlcat(x->d86_mnem, "16", OPLEN); + break; + } + } + + if (dp->it_avxsuf == AVS5Q) { + (void) strlcat(x->d86_mnem, vex_W != 0 ? "q" : "d", + OPLEN); + } +#endif +} + +/* + * The following three functions adjust the register selection based on any + * EVEX prefix bits present. See Intel 64 and IA-32 Architectures Software + * Developer’s Manual Volume 2 (IASDv2), section 2.6.1 Table 2-30 and + * section 2.6.2 Table 2-31. + */ +static void +dtrace_evex_adjust_reg(uint_t evex_byte1, uint_t *reg) +{ + if (reg != NULL) { + if ((VEX_R & evex_byte1) == 0) { + *reg += 8; + } + if ((EVEX_R & evex_byte1) == 0) { + *reg += 16; + } + } +} + +static void +dtrace_evex_adjust_rm(uint_t evex_byte1, uint_t *r_m) +{ + if (r_m != NULL) { + if ((VEX_B & evex_byte1) == 0) { + *r_m += 8; + } + if ((VEX_X & evex_byte1) == 0) { + *r_m += 16; + } + } +} + +/* + * Use evex_L to set wbit. See IASDv2 Section 2.6.10 and Table 2-36. + */ +static void +dtrace_evex_adjust_reg_name(uint_t evex_L, uint_t *wbitp) +{ + switch (evex_L) { + case 0x0: + *wbitp = XMM_OPND; + break; + case 0x1: + *wbitp = YMM_OPND; + break; + case 0x2: + *wbitp = ZMM_OPND; + break; + } +} + +/* + * Adjust operand value for disp8*N immediate. See IASDv2 Section 2.6.5. + * This currently only handles a subset of the possibilities. + */ +static void +dtrace_evex_adjust_disp8_n(dis86_t *x, int opindex, uint_t L, uint_t modrm) +{ + d86opnd_t *opnd = &x->d86_opnd[opindex]; + + if (x->d86_error) + return; + + /* Check disp8 bit in the ModR/M byte */ + if ((modrm & 0x80) == 0x80) + return; + + /* use evex_L to adjust the value */ + switch (L) { + case 0x0: + opnd->d86_value *= 16; + break; + case 0x1: + opnd->d86_value *= 32; + break; + case 0x2: + opnd->d86_value *= 64; + break; + } +} + +/* + * Adjust target for opmask and zeroing. See IASDv2 Section 2.6.1 Table 2-30. + */ +/* ARGSUSED */ +static void +dtrace_evex_adjust_z_opmask(dis86_t *x, uint_t tgtop, uint_t evex_byte3) +{ +#ifdef DIS_TEXT + char *opnd = x->d86_opnd[tgtop].d86_opnd; + int opmask_reg = evex_byte3 & EVEX_OPREG_MASK; +#endif + if (x->d86_error) + return; + +#ifdef DIS_TEXT + if (opmask_reg != 0) { + /* Append the opmask register to operand 1 */ + (void) strlcat(opnd, "{", OPLEN); + (void) strlcat(opnd, dis_KOPMASKREG[opmask_reg], OPLEN); + (void) strlcat(opnd, "}", OPLEN); + } + if ((evex_byte3 & EVEX_ZERO_MASK) != 0) { + /* Append the 'zeroing' modifier to operand 1 */ + (void) strlcat(opnd, "{z}", OPLEN); + } +#endif /* DIS_TEXT */ +} + +/* * Get an immediate operand of the given size, with sign extension. */ static void @@ -2528,6 +3255,7 @@ dtrace_imm_opnd(dis86_t *x, int wbit, int size, int opindex) case MM_OPND: case XMM_OPND: case YMM_OPND: + case ZMM_OPND: case SEG_OPND: case CONTROL_OPND: case DEBUG_OPND: @@ -2615,7 +3343,7 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) uint_t ss; /* scale-factor from opcode */ uint_t index; /* index register number */ uint_t base; /* base register number */ - int dispsize; /* size of displacement in bytes */ + int dispsize; /* size of displacement in bytes */ #ifdef DIS_TEXT char *opnd = x->d86_opnd[opindex].d86_opnd; #endif @@ -2641,6 +3369,12 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) case YMM_OPND: (void) strlcat(opnd, dis_YMMREG[r_m], OPLEN); break; + case ZMM_OPND: + (void) strlcat(opnd, dis_ZMMREG[r_m], OPLEN); + break; + case KOPMASK_OPND: + (void) strlcat(opnd, dis_KOPMASKREG[r_m], OPLEN); + break; case SEG_OPND: (void) strlcat(opnd, dis_SEGREG[r_m], OPLEN); break; @@ -2705,10 +3439,12 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) } /* - * 32 and 64 bit addressing modes are more complex since they - * can involve an SIB (scaled index and base) byte to decode. + * 32 and 64 bit addressing modes are more complex since they can + * involve an SIB (scaled index and base) byte to decode. When using VEX + * and EVEX encodings, the r_m indicator for a SIB may be offset by 8 + * and 24 (8 + 16) respectively. */ - if (r_m == ESP_REGNO || r_m == ESP_REGNO + 8) { + if (r_m == ESP_REGNO || r_m == ESP_REGNO + 8 || r_m == ESP_REGNO + 24) { have_SIB = 1; dtrace_get_SIB(x, &ss, &index, &base); if (x->d86_error) @@ -2778,10 +3514,13 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) regs = (char **)dis_REG64; if (x->d86_vsib != 0) { - if (wbit == YMM_OPND) /* NOTE this is not addr_size! */ + if (wbit == YMM_OPND) { /* NOTE this is not addr_size */ bregs = (char **)dis_YMMREG; - else + } else if (wbit == XMM_OPND) { bregs = (char **)dis_XMMREG; + } else { + bregs = (char **)dis_ZMMREG; + } sf = dis_vscale_factor; } else { bregs = regs; @@ -2845,7 +3584,7 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) /* * Similar, but for 2 operands plus an immediate. * vbit indicates direction - * 0 for "opcode imm, r, r_m" or + * 0 for "opcode imm, r, r_m" or * 1 for "opcode imm, r_m, r" */ #define THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, w2, immsize, vbit) { \ @@ -2890,7 +3629,7 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) int dtrace_disx86(dis86_t *x, uint_t cpu_mode) { - instable_t *dp; /* decode table being used */ + const instable_t *dp; /* decode table being used */ #ifdef DIS_TEXT uint_t i; #endif @@ -2916,6 +3655,8 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t opcode5; /* low nibble of 2nd byte */ uint_t opcode6; /* high nibble of 3rd byte */ uint_t opcode7; /* low nibble of 3rd byte */ + uint_t opcode8; /* high nibble of 4th byte */ + uint_t opcode9; /* low nibble of 4th byte */ uint_t opcode_bytes = 1; /* @@ -2942,6 +3683,13 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t vex_byte1 = 0; /* + * EVEX prefix byte 1 includes vex.r, vex.x, vex.b and evex.r. + */ + uint_t evex_byte1 = 0; + uint_t evex_byte2 = 0; + uint_t evex_byte3 = 0; + + /* * For 32-bit mode, it should prefetch the next byte to * distinguish between AVX and les/lds */ @@ -2954,7 +3702,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t vex_X = 1; uint_t vex_B = 1; uint_t vex_W = 0; - uint_t vex_L; + uint_t vex_L = 0; + uint_t evex_L = 0; + uint_t evex_modrm = 0; + uint_t evex_prefix = 0; dis_gather_regs_t *vreg; #ifdef DIS_TEXT @@ -3083,6 +3834,127 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) } } + /* + * The EVEX prefix and "bound" instruction share the same first byte. + * "bound" is only valid for 32-bit. For 64-bit this byte begins the + * EVEX prefix and the 2nd byte must have bits 2 & 3 set to 0. + */ + if (opcode1 == 0x6 && opcode2 == 0x2) { + evex_prefix = 0x62; + + /* + * An EVEX prefix is 4 bytes long, get the next 3 bytes. + */ + if (dtrace_get_opcode(x, &opcode4, &opcode5) != 0) + goto error; + + if (addr_size == SIZE32 && (opcode4 & 0xf) == 0) { + /* + * Upper bits in 2nd byte == 0 is 'bound' instn. + * + * We've already read the byte so perform the + * equivalent of dtrace_get_modrm on the byte and set + * the flag to indicate we've already read it. + */ + char b = (opcode4 << 4) | opcode5; + + r_m = b & 0x7; + reg = (b >> 3) & 0x7; + mode = (b >> 6) & 0x3; + vex_prefetch = 1; + goto not_avx512; + } + + /* check for correct bits being 0 in 2nd byte */ + if ((opcode5 & 0xc) != 0) + goto error; + + if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0) + goto error; + /* check for correct bit being 1 in 3rd byte */ + if ((opcode7 & 0x4) == 0) + goto error; + + if (dtrace_get_opcode(x, &opcode8, &opcode9) != 0) + goto error; + + /* Reuse opcode1 & opcode2 to get the real opcode now */ + if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0) + goto error; + + /* + * We only use the high nibble from the 2nd byte of the prefix + * and save it in the low bits of evex_byte1. This is because + * two of the bits in opcode5 are constant 0 (checked above), + * and the other two bits are captured in vex_m. Also, the VEX + * constants we check in evex_byte1 are against the low bits. + */ + evex_byte1 = opcode4; + evex_byte2 = (opcode6 << 4) | opcode7; + evex_byte3 = (opcode8 << 4) | opcode9; + + vex_m = opcode5 & EVEX_m; + vex_v = (((opcode6 << 4) | opcode7) & VEX_v) >> 3; + vex_W = (opcode6 & VEX_W) >> 3; + vex_p = opcode7 & VEX_p; + + /* + * Store the corresponding prefix information for later use when + * calculating the SIB. + */ + if ((evex_byte1 & VEX_R) == 0) + x->d86_rex_prefix |= REX_R; + if ((evex_byte1 & VEX_X) == 0) + x->d86_rex_prefix |= REX_X; + if ((evex_byte1 & VEX_B) == 0) + x->d86_rex_prefix |= REX_B; + + /* Currently only 3 valid values for evex L'L: 00, 01, 10 */ + evex_L = (opcode8 & EVEX_L) >> 1; + + switch (vex_p) { + case VEX_p_66: + switch (vex_m) { + case VEX_m_0F: + dp = &dis_opEVEX660F[(opcode1 << 4) | opcode2]; + break; + case VEX_m_0F38: + dp = &dis_opEVEX660F38[(opcode1 << 4) | + opcode2]; + break; + case VEX_m_0F3A: + dp = &dis_opEVEX660F3A[(opcode1 << 4) | + opcode2]; + break; + default: + goto error; + } + break; + case VEX_p_F3: + switch (vex_m) { + case VEX_m_0F: + dp = &dis_opEVEXF30F[(opcode1 << 4) | opcode2]; + break; + default: + goto error; + } + break; + case VEX_p_F2: + switch (vex_m) { + case VEX_m_0F: + dp = &dis_opEVEXF20F[(opcode1 << 4) | opcode2]; + break; + default: + goto error; + } + break; + default: + dp = &dis_opEVEX0F[(opcode1 << 4) | opcode2]; + break; + } + } +not_avx512: + if (vex_prefix == VEX_2bytes) { if (!vex_prefetch) { if (dtrace_get_opcode(x, &opcode3, &opcode4) != 0) @@ -3216,11 +4088,14 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (vex_prefix) { if (dp->it_vexwoxmm) { wbit = LONG_OPND; + } else if (dp->it_vexopmask) { + wbit = KOPMASK_OPND; } else { - if (vex_L) + if (vex_L) { wbit = YMM_OPND; - else + } else { wbit = XMM_OPND; + } } } @@ -3298,16 +4173,17 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (opnd_size_prefix == 0) { goto error; } + break; case XMMP_66o: if (opnd_size_prefix == 0) { /* SSSE3 MMX instructions */ dp_mmx = *dp; - dp = &dp_mmx; - dp->it_adrmode = MMOPM_66o; + dp_mmx.it_adrmode = MMOPM_66o; #ifdef DIS_MEM - dp->it_size = 8; + dp_mmx.it_size = 8; #endif + dp = &dp_mmx; } break; default: @@ -3388,11 +4264,11 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (opnd_size_prefix == 0) { /* SSSE3 MMX instructions */ dp_mmx = *dp; - dp = &dp_mmx; - dp->it_adrmode = MM; + dp_mmx.it_adrmode = MM; #ifdef DIS_MEM - dp->it_size = 8; + dp_mmx.it_size = 8; #endif + dp = &dp_mmx; } break; case CRC32: @@ -3409,6 +4285,9 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) default: goto error; } + } else if (rep_prefix == 0xf3 && opcode4 == 0 && opcode5 == 9) { + rep_prefix = 0; + dp = (instable_t *)&dis_opWbnoinvd; } else { dp = (instable_t *)&dis_op0F[opcode4][opcode5]; } @@ -3579,6 +4458,12 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) opnd_size_prefix = 0; if (opnd_size == SIZE16) opnd_size = SIZE32; + } else if (reg == 4 || reg == 5) { + /* + * We have xsavec (4) or xsaves (5), so rewrite. + */ + dp = (instable_t *)&dis_op0FC7[reg]; + break; } break; @@ -3615,8 +4500,8 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) /* * Calculate our offset in dis_op0F */ - if ((uintptr_t)dp - (uintptr_t)dis_op0F - > sizeof (dis_op0F)) + if ((uintptr_t)dp - (uintptr_t)dis_op0F > + sizeof (dis_op0F)) goto error; off = ((uintptr_t)dp - (uintptr_t)dis_op0F) / @@ -3629,6 +4514,19 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) } } break; + case FSGS: + if (rep_prefix == 0xf3) { + if ((uintptr_t)dp - (uintptr_t)dis_op0FAE > + sizeof (dis_op0FAE)) + goto error; + + off = ((uintptr_t)dp - (uintptr_t)dis_op0FAE) / + sizeof (instable_t); + dp = (instable_t *)&dis_opF30FAE[off]; + rep_prefix = 0; + } else if (rep_prefix != 0x00) { + goto error; + } } /* @@ -3660,9 +4558,49 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (strcmp(dp->it_name, "INVALID") == 0) goto error; (void) strlcat(x->d86_mnem, dp->it_name, OPLEN); - if (dp->it_avxsuf && dp->it_suffix) { + if (dp->it_avxsuf == AVS2 && dp->it_suffix) { (void) strlcat(x->d86_mnem, vex_W != 0 ? "q" : "d", OPLEN); + } else if (dp->it_vexopmask && dp->it_suffix) { + /* opmask instructions */ + + if (opcode1 == 4 && opcode2 == 0xb) { + /* It's a kunpck. */ + if (vex_prefix == VEX_2bytes) { + (void) strlcat(x->d86_mnem, + vex_p == 0 ? "wd" : "bw", OPLEN); + } else { + /* vex_prefix == VEX_3bytes */ + (void) strlcat(x->d86_mnem, + "dq", OPLEN); + } + } else if (opcode1 == 3) { + /* It's a kshift[l|r]. */ + if (vex_W == 0) { + (void) strlcat(x->d86_mnem, + opcode2 == 2 || + opcode2 == 0 ? + "b" : "d", OPLEN); + } else { + /* W == 1 */ + (void) strlcat(x->d86_mnem, + opcode2 == 3 || opcode2 == 1 ? + "q" : "w", OPLEN); + } + } else { + /* if (vex_prefix == VEX_2bytes) { */ + if ((cpu_mode == SIZE64 && opnd_size == 2) || + vex_prefix == VEX_2bytes) { + (void) strlcat(x->d86_mnem, + vex_p == 0 ? "w" : + vex_p == 1 ? "b" : "d", + OPLEN); + } else { + /* vex_prefix == VEX_3bytes */ + (void) strlcat(x->d86_mnem, + vex_p == 1 ? "d" : "q", OPLEN); + } + } } else if (dp->it_suffix) { char *types[] = {"", "w", "l", "q"}; if (opcode_bytes == 2 && opcode4 == 4) { @@ -3679,7 +4617,8 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) * To handle PINSRD and PEXTRD */ (void) strlcat(x->d86_mnem, "d", OPLEN); - } else { + } else if (dp != &dis_distable[0x6][0x2]) { + /* bound instructions (0x62) have no suffix */ (void) strlcat(x->d86_mnem, types[opnd_size], OPLEN); } @@ -3694,7 +4633,7 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) /* * In vex mode the rex_prefix has no meaning */ - if (!vex_prefix) + if (!vex_prefix && evex_prefix == 0) x->d86_rex_prefix = rex_prefix; x->d86_opnd_size = opnd_size; x->d86_addr_size = addr_size; @@ -3980,6 +4919,24 @@ just_mem: #endif NOMEM; break; + } else if (mode == 3 && r_m == 2) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "monitorx", OPLEN); +#endif + NOMEM; + break; + } else if (mode == 3 && r_m == 3) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "mwaitx", OPLEN); +#endif + NOMEM; + break; + } else if (mode == 3 && r_m == 4) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "clzero", OPLEN); +#endif + NOMEM; + break; } /*FALLTHROUGH*/ @@ -4221,6 +5178,8 @@ just_mem: case RM: case RM_66r: + if (vex_prefetch) + x->d86_got_modrm = 1; wbit = LONG_OPND; STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); break; @@ -4446,6 +5405,44 @@ xmmprm: x->d86_opnd[1] = x->d86_opnd[2]; x->d86_numopnds = 2; } + + /* + * The pclmulqdq instruction has a series of alternate names for + * various encodings of the immediate byte. As such, if we + * happen to find it and the immediate value matches, we'll + * rewrite the mnemonic. + */ + if (strcmp(dp->it_name, "pclmulqdq") == 0) { + boolean_t changed = B_TRUE; + switch (x->d86_opnd[0].d86_value) { + case 0x00: + (void) strncpy(x->d86_mnem, "pclmullqlqdq", + OPLEN); + break; + case 0x01: + (void) strncpy(x->d86_mnem, "pclmulhqlqdq", + OPLEN); + break; + case 0x10: + (void) strncpy(x->d86_mnem, "pclmullqhqdq", + OPLEN); + break; + case 0x11: + (void) strncpy(x->d86_mnem, "pclmulhqhqdq", + OPLEN); + break; + default: + changed = B_FALSE; + break; + } + + if (changed == B_TRUE) { + x->d86_opnd[0].d86_value_size = 0; + x->d86_opnd[0] = x->d86_opnd[1]; + x->d86_opnd[1] = x->d86_opnd[2]; + x->d86_numopnds = 2; + } + } #endif break; @@ -4702,22 +5699,44 @@ xmmprm: dtrace_get_modrm(x, &mode, ®, &r_m); /* sfence doesn't take operands */ + if (mode != REG_ONLY) { + if (opnd_size_prefix == 0x66) { #ifdef DIS_TEXT - if (mode == REG_ONLY) { - (void) strlcat(x->d86_mnem, "sfence", OPLEN); - } else { - (void) strlcat(x->d86_mnem, "clflush", OPLEN); + (void) strlcat(x->d86_mnem, "clflushopt", + OPLEN); +#endif + } else if (opnd_size_prefix == 0) { +#ifdef DIS_TEXT + (void) strlcat(x->d86_mnem, "clflush", OPLEN); +#endif + } else { + /* Unknown instruction */ + goto error; + } + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); dtrace_get_operand(x, mode, r_m, BYTE_OPND, 0); NOMEM; +#ifdef DIS_TEXT + } else { + (void) strlcat(x->d86_mnem, "sfence", OPLEN); +#endif } -#else - if (mode != REG_ONLY) { - dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); - dtrace_get_operand(x, mode, r_m, LONG_OPND, 0); + break; + + case FSGS: + /* + * The FSGSBASE instructions are taken only when the mode is set + * to registers. They share opcodes with instructions like + * fxrstor, stmxcsr, etc. We handle the repz prefix earlier. + */ + wbit = WBIT(opcode2); + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m); + dtrace_get_operand(x, mode, r_m, wbit, 0); + if (mode == REG_ONLY) { NOMEM; } -#endif break; /* @@ -4733,7 +5752,8 @@ xmmprm: case XMMFENCE: /* - * XRSTOR and LFENCE share the same opcode but differ in mode + * XRSTOR, XSAVEOPT and LFENCE share the same opcode but + * differ in mode and reg. */ dtrace_get_modrm(x, &mode, ®, &r_m); @@ -4741,15 +5761,29 @@ xmmprm: /* * Only the following exact byte sequences are allowed: * - * 0f ae e8 lfence - * 0f ae f0 mfence + * 0f ae e8 lfence + * 0f ae f0 mfence */ if ((uint8_t)x->d86_bytes[x->d86_len - 1] != 0xe8 && (uint8_t)x->d86_bytes[x->d86_len - 1] != 0xf0) goto error; } else { #ifdef DIS_TEXT - (void) strncpy(x->d86_mnem, "xrstor", OPLEN); + if (reg == 5) { + (void) strncpy(x->d86_mnem, "xrstor", OPLEN); + } else if (reg == 6) { + if (opnd_size_prefix == 0x66) { + (void) strncpy(x->d86_mnem, "clwb", + OPLEN); + } else if (opnd_size_prefix == 0x00) { + (void) strncpy(x->d86_mnem, "xsaveopt", + OPLEN); + } else { + goto error; + } + } else { + goto error; + } #endif dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); dtrace_get_operand(x, mode, r_m, BYTE_OPND, 0); @@ -5073,6 +6107,37 @@ L_VEX_MX: break; + case VEX_KMR: + /* opmask: mod_rm := %k */ + x->d86_numopnds = 2; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, mode, r_m, LONG_OPND, 1); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 0); + break; + + case VEX_KRM: + /* opmask: mod_reg := mod_rm */ + x->d86_numopnds = 2; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + if (mode == REG_ONLY) { + dtrace_get_operand(x, mode, r_m, KOPMASK_OPND, 0); + } else { + dtrace_get_operand(x, mode, r_m, LONG_OPND, 0); + } + break; + + case VEX_KRR: + /* opmask: mod_reg := mod_rm */ + x->d86_numopnds = 2; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + dtrace_get_operand(x, mode, reg, wbit, 1); + dtrace_get_operand(x, REG_ONLY, r_m, LONG_OPND, 0); + break; + case VEX_RRI: /* implicit(eflags/r32) := op(ModR/M.reg, ModR/M.rm) */ x->d86_numopnds = 2; @@ -5258,6 +6323,77 @@ L_VEX_RM: dtrace_get_operand(x, mode, r_m, wbit, 0); break; } + case EVEX_MX: + /* ModR/M.reg := op(ModR/M.rm) */ + x->d86_numopnds = 2; + dtrace_evex_mnem_adjust(x, dp, vex_W, evex_byte2); + dtrace_get_modrm(x, &mode, ®, &r_m); + evex_modrm = x->d86_bytes[x->d86_len - 1] & 0xff; + dtrace_evex_adjust_reg(evex_byte1, ®); + dtrace_evex_adjust_rm(evex_byte1, &r_m); + dtrace_evex_adjust_reg_name(evex_L, &wbit); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); + dtrace_evex_adjust_z_opmask(x, 1, evex_byte3); + dtrace_get_operand(x, mode, r_m, wbit, 0); + dtrace_evex_adjust_disp8_n(x, 0, evex_L, evex_modrm); + break; + case EVEX_RX: + /* ModR/M.rm := op(ModR/M.reg) */ + x->d86_numopnds = 2; + dtrace_evex_mnem_adjust(x, dp, vex_W, evex_byte2); + dtrace_get_modrm(x, &mode, ®, &r_m); + evex_modrm = x->d86_bytes[x->d86_len - 1] & 0xff; + dtrace_evex_adjust_reg(evex_byte1, ®); + dtrace_evex_adjust_rm(evex_byte1, &r_m); + dtrace_evex_adjust_reg_name(evex_L, &wbit); + dtrace_get_operand(x, mode, r_m, wbit, 1); + dtrace_evex_adjust_disp8_n(x, 1, evex_L, evex_modrm); + dtrace_evex_adjust_z_opmask(x, 1, evex_byte3); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 0); + break; + case EVEX_RMrX: + /* ModR/M.reg := op(EVEX.vvvv, ModR/M.r/m) */ + x->d86_numopnds = 3; + dtrace_evex_mnem_adjust(x, dp, vex_W, evex_byte2); + dtrace_get_modrm(x, &mode, ®, &r_m); + evex_modrm = x->d86_bytes[x->d86_len - 1] & 0xff; + dtrace_evex_adjust_reg(evex_byte1, ®); + dtrace_evex_adjust_rm(evex_byte1, &r_m); + dtrace_evex_adjust_reg_name(evex_L, &wbit); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + /* + * EVEX.vvvv is the same as VEX.vvvv (ones complement of the + * register specifier). The EVEX prefix handling uses the vex_v + * variable for these bits. + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + dtrace_evex_adjust_disp8_n(x, 0, evex_L, evex_modrm); + dtrace_evex_adjust_z_opmask(x, 2, evex_byte3); + break; + case EVEX_RMRX: + /* ModR/M.reg := op(EVEX.vvvv, ModR/M.r_m, imm8) */ + x->d86_numopnds = 4; + + dtrace_evex_mnem_adjust(x, dp, vex_W, evex_byte2); + dtrace_get_modrm(x, &mode, ®, &r_m); + evex_modrm = x->d86_bytes[x->d86_len - 1] & 0xff; + dtrace_evex_adjust_reg(evex_byte1, ®); + dtrace_evex_adjust_rm(evex_byte1, &r_m); + dtrace_evex_adjust_reg_name(evex_L, &wbit); + dtrace_get_operand(x, REG_ONLY, reg, wbit, 3); + /* + * EVEX.vvvv is the same as VEX.vvvv (ones complement of the + * register specifier). The EVEX prefix handling uses the vex_v + * variable for these bits. + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 2); + dtrace_get_operand(x, mode, r_m, wbit, 1); + dtrace_evex_adjust_disp8_n(x, 0, evex_L, evex_modrm); + dtrace_evex_adjust_z_opmask(x, 3, evex_byte3); + + dtrace_imm_opnd(x, wbit, 1, 0); + break; /* an invalid op code */ case AM: case DM: @@ -5528,7 +6664,6 @@ dtrace_disx86_str(dis86_t *dis, uint_t mode, uint64_t pc, char *buf, save_mask = mask; } (void) strlcat(buf, op->d86_opnd, buflen); - break; case MODE_IPREL: diff --git a/sys/cddl/dev/dtrace/x86/dis_tables.h b/sys/cddl/dev/dtrace/x86/dis_tables.h index 87b006cf7845..fa0aa5e9430d 100644 --- a/sys/cddl/dev/dtrace/x86/dis_tables.h +++ b/sys/cddl/dev/dtrace/x86/dis_tables.h @@ -27,7 +27,6 @@ /* All Rights Reserved */ /* - * $FreeBSD$ */ #ifndef _DIS_TABLES_H diff --git a/sys/cddl/dev/dtrace/x86/instr_size.c b/sys/cddl/dev/dtrace/x86/instr_size.c index 6eea987244b1..4397b3bd69d7 100644 --- a/sys/cddl/dev/dtrace/x86/instr_size.c +++ b/sys/cddl/dev/dtrace/x86/instr_size.c @@ -18,8 +18,6 @@ * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END - * - * $FreeBSD$ */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. @@ -29,11 +27,6 @@ /* Copyright (c) 1988 AT&T */ /* All Rights Reserved */ - -#ifdef illumos -#pragma ident "@(#)instr_size.c 1.14 05/07/08 SMI" -#endif - #include <sys/types.h> #include <sys/param.h> #include <sys/proc.h> @@ -49,8 +42,9 @@ typedef u_int model_t; #define DATAMODEL_NATIVE 0 -int dtrace_instr_size(uchar_t *); -int dtrace_instr_size_isa(uchar_t *, model_t, int *); +int dtrace_dis_get_byte(void *); +int dtrace_instr_size(uint8_t *); +int dtrace_instr_size_isa(uint8_t *, model_t, int *); #endif #include <dis_tables.h> @@ -79,11 +73,11 @@ typedef enum dis_isize { /* * get a byte from instruction stream */ -static int +int dtrace_dis_get_byte(void *p) { int ret; - uchar_t **instr = p; + uint8_t **instr = p; ret = **instr; *instr += 1; @@ -101,7 +95,7 @@ dtrace_dis_get_byte(void *p) */ /* ARGSUSED2 */ static int -dtrace_dis_isize(uchar_t *instr, dis_isize_t which, model_t model, int *rmindex) +dtrace_dis_isize(uint8_t *instr, dis_isize_t which, model_t model, int *rmindex) { int sz; dis86_t x; @@ -127,13 +121,13 @@ dtrace_dis_isize(uchar_t *instr, dis_isize_t which, model_t model, int *rmindex) } int -dtrace_instr_size_isa(uchar_t *instr, model_t model, int *rmindex) +dtrace_instr_size_isa(uint8_t *instr, model_t model, int *rmindex) { return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, model, rmindex)); } int -dtrace_instr_size(uchar_t *instr) +dtrace_instr_size(uint8_t *instr) { return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, DATAMODEL_NATIVE, NULL)); diff --git a/sys/cddl/dev/dtrace/x86/regset.h b/sys/cddl/dev/dtrace/x86/regset.h index ad12e26a5aec..8b187826ea10 100644 --- a/sys/cddl/dev/dtrace/x86/regset.h +++ b/sys/cddl/dev/dtrace/x86/regset.h @@ -19,7 +19,6 @@ * * CDDL HEADER END * - * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. @@ -35,7 +34,6 @@ #define _REGSET_H /* - * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" */ #ifdef __cplusplus @@ -61,35 +59,6 @@ extern "C" { #define REG_GSBASE 27 #define REG_FSBASE 26 -#ifdef illumos -#define REG_DS 25 -#define REG_ES 24 - -#define REG_GS 23 -#define REG_FS 22 -#define REG_SS 21 -#define REG_RSP 20 -#define REG_RFL 19 -#define REG_CS 18 -#define REG_RIP 17 -#define REG_ERR 16 -#define REG_TRAPNO 15 -#define REG_RAX 14 -#define REG_RCX 13 -#define REG_RDX 12 -#define REG_RBX 11 -#define REG_RBP 10 -#define REG_RSI 9 -#define REG_RDI 8 -#define REG_R8 7 -#define REG_R9 6 -#define REG_R10 5 -#define REG_R11 4 -#define REG_R12 3 -#define REG_R13 2 -#define REG_R14 1 -#define REG_R15 0 -#else /* !illumos */ #define REG_SS 25 #define REG_RSP 24 #define REG_RFL 23 @@ -116,33 +85,11 @@ extern "C" { #define REG_R13 2 #define REG_R14 1 #define REG_R15 0 -#endif /* illumos */ /* * The names and offsets defined here are specified by i386 ABI suppl. */ -#ifdef illumos -#define SS 18 /* only stored on a privilege transition */ -#define UESP 17 /* only stored on a privilege transition */ -#define EFL 16 -#define CS 15 -#define EIP 14 -#define ERR 13 -#define TRAPNO 12 -#define EAX 11 -#define ECX 10 -#define EDX 9 -#define EBX 8 -#define ESP 7 -#define EBP 6 -#define ESI 5 -#define EDI 4 -#define DS 3 -#define ES 2 -#define FS 1 -#define GS 0 -#else /* !illumos */ #define GS 18 #define SS 17 /* only stored on a privilege transition */ #define UESP 16 /* only stored on a privilege transition */ @@ -162,7 +109,6 @@ extern "C" { #define DS 2 #define ES 1 #define FS 0 -#endif /* illumos */ #define REG_PC EIP #define REG_FP EBP diff --git a/sys/cddl/dev/fbt/aarch64/fbt_isa.c b/sys/cddl/dev/fbt/aarch64/fbt_isa.c index 07f02e2edb72..4f6d28c2f32b 100644 --- a/sys/cddl/dev/fbt/aarch64/fbt_isa.c +++ b/sys/cddl/dev/fbt/aarch64/fbt_isa.c @@ -22,8 +22,6 @@ * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * Portions Copyright 2015 Ruslan Bukin <br@bsdpad.com> - * - * $FreeBSD$ */ /* @@ -31,19 +29,13 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/dtrace.h> #include "fbt.h" -#define AARCH64_BRK 0xd4200000 -#define AARCH64_BRK_IMM16_SHIFT 5 -#define AARCH64_BRK_IMM16_VAL (0x40d << AARCH64_BRK_IMM16_SHIFT) -#define FBT_PATCHVAL (AARCH64_BRK | AARCH64_BRK_IMM16_VAL) -#define FBT_ENTRY "entry" -#define FBT_RETURN "return" +#define FBT_PATCHVAL DTRACE_PATCHVAL #define FBT_AFRAMES 4 int @@ -79,13 +71,13 @@ fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { - vm_offset_t addr; + void *addr; - if (!arm64_get_writable_addr((vm_offset_t)fbt->fbtp_patchpoint, &addr)) + if (!arm64_get_writable_addr(fbt->fbtp_patchpoint, &addr)) panic("%s: Unable to write new instruction", __func__); *(fbt_patchval_t *)addr = val; - cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); + cpu_icache_sync_range(fbt->fbtp_patchpoint, 4); } int @@ -97,7 +89,6 @@ fbt_provide_module_function(linker_file_t lf, int symindx, uint32_t *instr, *limit; const char *name; char *modname; - bool found; int offs; modname = opaque; @@ -126,47 +117,37 @@ fbt_provide_module_function(linker_file_t lf, int symindx, if ((*instr & BTI_MASK) == BTI_INSTR) instr++; - /* Look for stp (pre-indexed) operation */ - found = false; /* * If the first instruction is a nop it's a specially marked * asm function. We only support a nop first as it's not a normal * part of the function prologue. */ if (*instr == NOP_INSTR) - found = true; - if (!found) { - for (; instr < limit; instr++) { - /* - * Some functions start with - * "stp xt1, xt2, [xn, <const>]!" - */ - if ((*instr & LDP_STP_MASK) == STP_64) { - /* - * Assume any other store of this type means we - * are past the function prolog. - */ - if (((*instr >> ADDR_SHIFT) & ADDR_MASK) == 31) - found = true; - break; - } + goto found; + /* Look for stp (pre-indexed) or sub operation */ + for (; instr < limit; instr++) { + /* + * Functions start with "stp xt1, xt2, [xn, <const>]!" or + * "sub sp, sp, <const>". + * + * Sometimes the compiler will have a sub instruction that is + * not of the above type so don't stop if we see one. + */ + if ((*instr & LDP_STP_MASK) == STP_64) { /* - * Some functions start with a "sub sp, sp, <const>" - * Sometimes the compiler will have a sub instruction - * that is not of the above type so don't stop if we - * see one. + * Assume any other store of this type means we are + * past the function prologue. */ - if ((*instr & SUB_MASK) == SUB_INSTR && - ((*instr >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 && - ((*instr >> SUB_RN_SHIFT) & SUB_R_MASK) == 31) { - found = true; + if (((*instr >> ADDR_SHIFT) & ADDR_MASK) == 31) break; - } - } + } else if ((*instr & SUB_MASK) == SUB_INSTR && + ((*instr >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 && + ((*instr >> SUB_RN_SHIFT) & SUB_R_MASK) == 31) + break; } - - if (!found) +found: + if (instr >= limit) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); diff --git a/sys/cddl/dev/fbt/aarch64/fbt_isa.h b/sys/cddl/dev/fbt/aarch64/fbt_isa.h index 5552f31a64a6..e06f7a4b3e74 100644 --- a/sys/cddl/dev/fbt/aarch64/fbt_isa.h +++ b/sys/cddl/dev/fbt/aarch64/fbt_isa.h @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef _FBT_ISA_H_ diff --git a/sys/cddl/dev/fbt/arm/fbt_isa.c b/sys/cddl/dev/fbt/arm/fbt_isa.c index 0be28b56aa6a..be16533de18c 100644 --- a/sys/cddl/dev/fbt/arm/fbt_isa.c +++ b/sys/cddl/dev/fbt/arm/fbt_isa.c @@ -22,8 +22,6 @@ * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * - * $FreeBSD$ - * */ /* @@ -31,7 +29,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/dtrace.h> @@ -45,9 +42,6 @@ #define FBT_JUMP 0xea000000 #define FBT_SUBSP 0xe24dd000 -#define FBT_ENTRY "entry" -#define FBT_RETURN "return" - int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { diff --git a/sys/cddl/dev/fbt/arm/fbt_isa.h b/sys/cddl/dev/fbt/arm/fbt_isa.h index 5552f31a64a6..e06f7a4b3e74 100644 --- a/sys/cddl/dev/fbt/arm/fbt_isa.h +++ b/sys/cddl/dev/fbt/arm/fbt_isa.h @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef _FBT_ISA_H_ diff --git a/sys/cddl/dev/fbt/fbt.c b/sys/cddl/dev/fbt/fbt.c index 35db8938d3c3..481c896e9775 100644 --- a/sys/cddl/dev/fbt/fbt.c +++ b/sys/cddl/dev/fbt/fbt.c @@ -20,8 +20,6 @@ * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * - * $FreeBSD$ - * */ /* @@ -29,7 +27,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/conf.h> diff --git a/sys/cddl/dev/fbt/fbt.h b/sys/cddl/dev/fbt/fbt.h index aa9bce564fa0..51540df1650d 100644 --- a/sys/cddl/dev/fbt/fbt.h +++ b/sys/cddl/dev/fbt/fbt.h @@ -20,8 +20,6 @@ * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * - * $FreeBSD$ - * */ /* @@ -34,6 +32,9 @@ #include "fbt_isa.h" +#define FBT_ENTRY "entry" +#define FBT_RETURN "return" + /* * fbt_probe is a bit of a misnomer. One of these structures is created for * each trace point of an FBT probe. A probe might have multiple trace points diff --git a/sys/cddl/dev/fbt/powerpc/fbt_isa.c b/sys/cddl/dev/fbt/powerpc/fbt_isa.c index 0da74c9cf076..d0b3ccdeb906 100644 --- a/sys/cddl/dev/fbt/powerpc/fbt_isa.c +++ b/sys/cddl/dev/fbt/powerpc/fbt_isa.c @@ -21,8 +21,6 @@ * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * - * $FreeBSD$ - * */ /* @@ -30,7 +28,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/dtrace.h> #include <machine/md_var.h> @@ -46,9 +43,7 @@ #define FBT_BR_MASK 0x03fffffc #define FBT_IS_JUMP(instr) ((instr & ~FBT_BR_MASK) == FBT_BRANCH) -#define FBT_ENTRY "entry" -#define FBT_RETURN "return" -#define FBT_AFRAMES 7 +#define FBT_AFRAMES 5 int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) diff --git a/sys/cddl/dev/fbt/powerpc/fbt_isa.h b/sys/cddl/dev/fbt/powerpc/fbt_isa.h index 5552f31a64a6..e06f7a4b3e74 100644 --- a/sys/cddl/dev/fbt/powerpc/fbt_isa.h +++ b/sys/cddl/dev/fbt/powerpc/fbt_isa.h @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef _FBT_ISA_H_ diff --git a/sys/cddl/dev/fbt/riscv/fbt_isa.c b/sys/cddl/dev/fbt/riscv/fbt_isa.c index 659a9d44c81c..1a3bd14ac1cf 100644 --- a/sys/cddl/dev/fbt/riscv/fbt_isa.c +++ b/sys/cddl/dev/fbt/riscv/fbt_isa.c @@ -22,8 +22,6 @@ * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * Portions Copyright 2016-2018 Ruslan Bukin <br@bsdpad.com> - * - * $FreeBSD$ */ /* @@ -31,7 +29,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/dtrace.h> @@ -43,8 +40,7 @@ #define FBT_C_PATCHVAL MATCH_C_EBREAK #define FBT_PATCHVAL MATCH_EBREAK -#define FBT_ENTRY "entry" -#define FBT_RETURN "return" +#define FBT_AFRAMES 5 int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) @@ -57,7 +53,7 @@ fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { - cpu->cpu_dtrace_caller = addr; + cpu->cpu_dtrace_caller = frame->tf_ra - INSN_SIZE; if (fbt->fbtp_roffset == 0) { dtrace_probe(fbt->fbtp_id, frame->tf_a[0], @@ -92,52 +88,6 @@ fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) }; } -static int -match_opcode(uint32_t insn, int match, int mask) -{ - - if (((insn ^ match) & mask) == 0) - return (1); - - return (0); -} - -static int -check_c_ret(uint32_t **instr) -{ - uint16_t *instr1; - int i; - - for (i = 0; i < 2; i++) { - instr1 = (uint16_t *)(*instr) + i; - if (match_opcode(*instr1, (MATCH_C_JR | (X_RA << RD_SHIFT)), - (MASK_C_JR | RD_MASK))) { - *instr = (uint32_t *)instr1; - return (1); - } - } - - return (0); -} - -static int -check_c_sdsp(uint32_t **instr) -{ - uint16_t *instr1; - int i; - - for (i = 0; i < 2; i++) { - instr1 = (uint16_t *)(*instr) + i; - if (match_opcode(*instr1, (MATCH_C_SDSP | RS2_C_RA), - (MASK_C_SDSP | RS2_C_MASK))) { - *instr = (uint32_t *)instr1; - return (1); - } - } - - return (0); -} - int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) @@ -175,15 +125,14 @@ fbt_provide_module_function(linker_file_t lf, int symindx, /* Look for sd operation */ for (; instr < limit; instr++) { /* Look for a non-compressed store of ra to sp */ - if (match_opcode(*instr, (MATCH_SD | RS2_RA | RS1_SP), - (MASK_SD | RS2_MASK | RS1_MASK))) { + if (dtrace_instr_sdsp(&instr)) { rval = DTRACE_INVOP_SD; patchval = FBT_PATCHVAL; break; } /* Look for a 'C'-compressed store of ra to sp. */ - if (check_c_sdsp(&instr)) { + if (dtrace_instr_c_sdsp(&instr)) { rval = DTRACE_INVOP_C_SDSP; patchval = FBT_C_PATCHVAL; break; @@ -196,7 +145,7 @@ fbt_provide_module_function(linker_file_t lf, int symindx, fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, - name, FBT_ENTRY, 3, fbt); + name, FBT_ENTRY, FBT_AFRAMES, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; @@ -214,15 +163,14 @@ fbt_provide_module_function(linker_file_t lf, int symindx, again: for (; instr < limit; instr++) { /* Look for non-compressed return */ - if (match_opcode(*instr, (MATCH_JALR | (X_RA << RS1_SHIFT)), - (MASK_JALR | RD_MASK | RS1_MASK | IMM_MASK))) { + if (dtrace_instr_ret(&instr)) { rval = DTRACE_INVOP_RET; patchval = FBT_PATCHVAL; break; } /* Look for 'C'-compressed return */ - if (check_c_ret(&instr)) { + if (dtrace_instr_c_ret(&instr)) { rval = DTRACE_INVOP_C_RET; patchval = FBT_C_PATCHVAL; break; @@ -239,7 +187,7 @@ again: fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, - name, FBT_RETURN, 3, fbt); + name, FBT_RETURN, FBT_AFRAMES, fbt); } else { retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; diff --git a/sys/cddl/dev/fbt/riscv/fbt_isa.h b/sys/cddl/dev/fbt/riscv/fbt_isa.h index 5552f31a64a6..e06f7a4b3e74 100644 --- a/sys/cddl/dev/fbt/riscv/fbt_isa.h +++ b/sys/cddl/dev/fbt/riscv/fbt_isa.h @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef _FBT_ISA_H_ diff --git a/sys/cddl/dev/fbt/x86/fbt_isa.c b/sys/cddl/dev/fbt/x86/fbt_isa.c index 74de00a3f00b..387a3f1582c3 100644 --- a/sys/cddl/dev/fbt/x86/fbt_isa.c +++ b/sys/cddl/dev/fbt/x86/fbt_isa.c @@ -20,8 +20,6 @@ * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * - * $FreeBSD$ - * */ /* @@ -29,7 +27,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/dtrace.h> @@ -57,23 +54,24 @@ #define FBT_PATCHVAL 0xf0 #endif -#define FBT_ENTRY "entry" -#define FBT_RETURN "return" +#define FBT_AFRAMES 2 int -fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) +fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch __unused) { solaris_cpu_t *cpu; uintptr_t *stack; - uintptr_t arg0, arg1, arg2, arg3, arg4; + uintptr_t arg0, arg1, arg2, arg3, arg4, rval; fbt_probe_t *fbt; int8_t fbtrval; #ifdef __amd64__ stack = (uintptr_t *)frame->tf_rsp; + rval = frame->tf_rax; #else /* Skip hardware-saved registers. */ stack = (uintptr_t *)frame->tf_isp + 3; + rval = frame->tf_eax; #endif cpu = &solaris_cpu[curcpu]; @@ -82,6 +80,16 @@ fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) if ((uintptr_t)fbt->fbtp_patchpoint != addr) continue; fbtrval = fbt->fbtp_rval; + + /* + * Report the address of the breakpoint for the benefit + * of consumers fetching register values with regs[]. + */ +#ifdef __i386__ + frame->tf_eip--; +#else + frame->tf_rip--; +#endif for (; fbt != NULL; fbt = fbt->fbtp_tracenext) { ASSERT(fbt->fbtp_rval == fbtrval); if (fbt->fbtp_roffset == 0) { @@ -141,6 +149,12 @@ fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) cpu->cpu_dtrace_caller = 0; } } + /* Advance to the instruction following the breakpoint. */ +#ifdef __i386__ + frame->tf_eip++; +#else + frame->tf_rip++; +#endif return (fbtrval); } @@ -219,7 +233,7 @@ fbt_provide_module_function(linker_file_t lf, int symindx, fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, - name, FBT_ENTRY, 3, fbt); + name, FBT_ENTRY, FBT_AFRAMES, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; @@ -313,7 +327,7 @@ again: if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, - name, FBT_RETURN, 3, fbt); + name, FBT_RETURN, FBT_AFRAMES, fbt); } else { retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; diff --git a/sys/cddl/dev/fbt/x86/fbt_isa.h b/sys/cddl/dev/fbt/x86/fbt_isa.h index 79190dbf9307..bcea8ceb915c 100644 --- a/sys/cddl/dev/fbt/x86/fbt_isa.h +++ b/sys/cddl/dev/fbt/x86/fbt_isa.h @@ -18,8 +18,6 @@ * * CDDL HEADER END * - * $FreeBSD$ - * */ #ifndef _FBT_ISA_H_ diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.c b/sys/cddl/dev/kinst/aarch64/kinst_isa.c new file mode 100644 index 000000000000..20ca26219a55 --- /dev/null +++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.c @@ -0,0 +1,453 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> + * Copyright (c) 2022 Mark Johnston <markj@FreeBSD.org> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> + +#include <sys/dtrace.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> + +#include "kinst.h" + +DPCPU_DEFINE_STATIC(struct kinst_cpu_state, kinst_state); + +static int +kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) +{ + kinst_patchval_t instr = kp->kp_savedval; + uint64_t imm; + uint8_t cond, reg, bitpos; + bool res; + + if (((instr >> 24) & 0x1f) == 0b10000) { + /* adr/adrp */ + reg = instr & 0x1f; + imm = (instr >> 29) & 0x3; + imm |= ((instr >> 5) & 0x0007ffff) << 2; + if (((instr >> 31) & 0x1) == 0) { + /* adr */ + if (imm & 0x0000000000100000) + imm |= 0xfffffffffff00000; + frame->tf_x[reg] = frame->tf_elr + imm; + } else { + /* adrp */ + imm <<= 12; + if (imm & 0x0000000100000000) + imm |= 0xffffffff00000000; + frame->tf_x[reg] = (frame->tf_elr & ~0xfff) + imm; + } + frame->tf_elr += INSN_SIZE; + } else if (((instr >> 26) & 0x3f) == 0b000101) { + /* b */ + imm = instr & 0x03ffffff; + if (imm & 0x0000000002000000) + imm |= 0xfffffffffe000000; + frame->tf_elr += imm << 2; + } else if (((instr >> 24) & 0xff) == 0b01010100) { + /* b.cond */ + imm = (instr >> 5) & 0x0007ffff; + if (imm & 0x0000000000040000) + imm |= 0xfffffffffffc0000; + cond = instr & 0xf; + switch ((cond >> 1) & 0x7) { + case 0b000: /* eq/ne */ + res = (frame->tf_spsr & PSR_Z) != 0; + break; + case 0b001: /* cs/cc */ + res = (frame->tf_spsr & PSR_C) != 0; + break; + case 0b010: /* mi/pl */ + res = (frame->tf_spsr & PSR_N) != 0; + break; + case 0b011: /* vs/vc */ + res = (frame->tf_spsr & PSR_V) != 0; + break; + case 0b100: /* hi/ls */ + res = ((frame->tf_spsr & PSR_C) != 0) && + ((frame->tf_spsr & PSR_Z) == 0); + break; + case 0b101: /* ge/lt */ + res = ((frame->tf_spsr & PSR_N) != 0) == + ((frame->tf_spsr & PSR_V) != 0); + break; + case 0b110: /* gt/le */ + res = ((frame->tf_spsr & PSR_Z) == 0) && + (((frame->tf_spsr & PSR_N) != 0) == + ((frame->tf_spsr & PSR_V) != 0)); + break; + case 0b111: /* al */ + res = 1; + break; + } + if ((cond & 0x1) && cond != 0b1111) + res = !res; + if (res) + frame->tf_elr += imm << 2; + else + frame->tf_elr += INSN_SIZE; + } else if (((instr >> 26) & 0x3f) == 0b100101) { + /* bl */ + imm = instr & 0x03ffffff; + if (imm & 0x0000000002000000) + imm |= 0xfffffffffe000000; + frame->tf_lr = frame->tf_elr + INSN_SIZE; + frame->tf_elr += imm << 2; + } else if (((instr >> 25) & 0x3f) == 0b011010) { + /* cbnz/cbz */ + cond = (instr >> 24) & 0x1; + reg = instr & 0x1f; + imm = (instr >> 5) & 0x0007ffff; + if (imm & 0x0000000000040000) + imm |= 0xfffffffffffc0000; + if (cond == 1 && frame->tf_x[reg] != 0) + /* cbnz */ + frame->tf_elr += imm << 2; + else if (cond == 0 && frame->tf_x[reg] == 0) + /* cbz */ + frame->tf_elr += imm << 2; + else + frame->tf_elr += INSN_SIZE; + } else if (((instr >> 25) & 0x3f) == 0b011011) { + /* tbnz/tbz */ + cond = (instr >> 24) & 0x1; + reg = instr & 0x1f; + bitpos = (instr >> 19) & 0x1f; + bitpos |= ((instr >> 31) & 0x1) << 5; + imm = (instr >> 5) & 0x3fff; + if (imm & 0x0000000000002000) + imm |= 0xffffffffffffe000; + if (cond == 1 && (frame->tf_x[reg] & (1 << bitpos)) != 0) + /* tbnz */ + frame->tf_elr += imm << 2; + else if (cond == 0 && (frame->tf_x[reg] & (1 << bitpos)) == 0) + /* tbz */ + frame->tf_elr += imm << 2; + else + frame->tf_elr += INSN_SIZE; + } + + return (0); +} + +static int +kinst_jump_next_instr(struct trapframe *frame, const struct kinst_probe *kp) +{ + frame->tf_elr = (register_t)((const uint8_t *)kp->kp_patchpoint + + INSN_SIZE); + + return (0); +} + +static void +kinst_trampoline_populate(struct kinst_probe *kp) +{ + static uint32_t bpt = KINST_PATCHVAL; + + kinst_memcpy(kp->kp_tramp, &kp->kp_savedval, INSN_SIZE); + kinst_memcpy(&kp->kp_tramp[INSN_SIZE], &bpt, INSN_SIZE); + + cpu_icache_sync_range(kp->kp_tramp, KINST_TRAMP_SIZE); +} + +/* + * There are two ways by which an instruction is traced: + * + * - By using the trampoline. + * - By emulating it in software (see kinst_emulate()). + * + * The trampoline is used for instructions that can be copied and executed + * as-is without additional modification. However, instructions that use + * PC-relative addressing have to be emulated, because ARM64 doesn't allow + * encoding of large displacements in a single instruction, and since we cannot + * clobber a register in order to encode the two-instruction sequence needed to + * create large displacements, we cannot use the trampoline at all. + * Fortunately, the instructions are simple enough to be emulated in just a few + * lines of code. + * + * The problem discussed above also means that, unlike amd64, we cannot encode + * a far-jump back from the trampoline to the next instruction. The mechanism + * employed to achieve this functionality, is to use a breakpoint instead of a + * jump after the copied instruction. This breakpoint is detected and handled + * by kinst_invop(), which performs the jump back to the next instruction + * manually (see kinst_jump_next_instr()). + */ +int +kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) +{ + solaris_cpu_t *cpu; + struct kinst_cpu_state *ks; + const struct kinst_probe *kp; + + ks = DPCPU_PTR(kinst_state); + + /* + * Detect if the breakpoint was triggered by the trampoline, and + * manually set the PC to the next instruction. + */ + if (ks->state == KINST_PROBE_FIRED && + addr == (uintptr_t)(ks->kp->kp_tramp + INSN_SIZE)) { + /* + * Restore interrupts if they were enabled prior to the first + * breakpoint. + */ + if ((ks->status & PSR_I) == 0) + frame->tf_spsr &= ~PSR_I; + ks->state = KINST_PROBE_ARMED; + return (kinst_jump_next_instr(frame, ks->kp)); + } + + LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { + if ((uintptr_t)kp->kp_patchpoint == addr) + break; + } + if (kp == NULL) + return (0); + + cpu = &solaris_cpu[curcpu]; + cpu->cpu_dtrace_caller = addr; + dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); + cpu->cpu_dtrace_caller = 0; + + if (kp->kp_md.emulate) + return (kinst_emulate(frame, kp)); + + ks->state = KINST_PROBE_FIRED; + ks->kp = kp; + + /* + * Cache the current SPSR and clear interrupts for the duration + * of the double breakpoint. + */ + ks->status = frame->tf_spsr; + frame->tf_spsr |= PSR_I; + frame->tf_elr = (register_t)kp->kp_tramp; + + return (0); +} + +void +kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) +{ + void *addr; + + if (!arm64_get_writable_addr(kp->kp_patchpoint, &addr)) + panic("%s: Unable to write new instruction", __func__); + *(kinst_patchval_t *)addr = val; + cpu_icache_sync_range(kp->kp_patchpoint, INSN_SIZE); +} + +static void +kinst_instr_dissect(struct kinst_probe *kp) +{ + struct kinst_probe_md *kpmd; + kinst_patchval_t instr = kp->kp_savedval; + + kpmd = &kp->kp_md; + kpmd->emulate = false; + + if (((instr >> 24) & 0x1f) == 0b10000) + kpmd->emulate = true; /* adr/adrp */ + else if (((instr >> 26) & 0x3f) == 0b000101) + kpmd->emulate = true; /* b */ + else if (((instr >> 24) & 0xff) == 0b01010100) + kpmd->emulate = true; /* b.cond */ + else if (((instr >> 26) & 0x3f) == 0b100101) + kpmd->emulate = true; /* bl */ + else if (((instr >> 25) & 0x3f) == 0b011010) + kpmd->emulate = true; /* cbnz/cbz */ + else if (((instr >> 25) & 0x3f) == 0b011011) + kpmd->emulate = true; /* tbnz/tbz */ + + if (!kpmd->emulate) + kinst_trampoline_populate(kp); +} + +static bool +kinst_instr_ldx(kinst_patchval_t instr) +{ + if (((instr >> 22) & 0xff) == 0b00100001) + return (true); + + return (false); +} + +static bool +kinst_instr_stx(kinst_patchval_t instr) +{ + if (((instr >> 22) & 0xff) == 0b00100000) + return (true); + + return (false); +} + +int +kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, + void *opaque) +{ + struct kinst_probe *kp; + dtrace_kinst_probedesc_t *pd; + const char *func; + kinst_patchval_t *instr, *limit, *tmp; + int n, off; + bool ldxstx_block, found; + + pd = opaque; + func = symval->name; + + if (kinst_excluded(func)) + return (0); + if (strcmp(func, pd->kpd_func) != 0) + return (0); + + instr = (kinst_patchval_t *)(symval->value); + limit = (kinst_patchval_t *)(symval->value + symval->size); + if (instr >= limit) + return (0); + + tmp = instr; + + /* + * Ignore any bti instruction at the start of the function + * we need to keep it there for any indirect branches calling + * the function on Armv8.5+ + */ + if ((*tmp & BTI_MASK) == BTI_INSTR) + tmp++; + + /* Look for stp (pre-indexed) operation */ + found = false; + + /* + * If the first instruction is a nop it's a specially marked + * asm function. We only support a nop first as it's not a normal + * part of the function prologue. + */ + if (*tmp == NOP_INSTR) + found = true; + for (; !found && tmp < limit; tmp++) { + /* + * Functions start with "stp xt1, xt2, [xn, <const>]!" or + * "sub sp, sp, <const>". + * + * Sometimes the compiler will have a sub instruction that is + * not of the above type so don't stop if we see one. + */ + if ((*tmp & LDP_STP_MASK) == STP_64) { + /* + * Assume any other store of this type means we are + * past the function prolog. + */ + if (((*tmp >> ADDR_SHIFT) & ADDR_MASK) == 31) + found = true; + } else if ((*tmp & SUB_MASK) == SUB_INSTR && + ((*tmp >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 && + ((*tmp >> SUB_RN_SHIFT) & SUB_R_MASK) == 31) + found = true; + } + + if (!found) + return (0); + + ldxstx_block = false; + for (n = 0; instr < limit; instr++) { + off = (int)((uint8_t *)instr - (uint8_t *)symval->value); + + /* + * Skip LDX/STX blocks that contain atomic operations. If a + * breakpoint is placed in a LDX/STX block, we violate the + * operation and the loop might fail. + */ + if (kinst_instr_ldx(*instr)) + ldxstx_block = true; + else if (kinst_instr_stx(*instr)) { + ldxstx_block = false; + continue; + } + if (ldxstx_block) + continue; + + /* + * XXX: Skip ADR and ADRP instructions. The arm64 exception + * handler has a micro-optimization where it doesn't restore + * callee-saved registers when returning from exceptions in + * EL1. This results in a panic when the kinst emulation code + * modifies one of those registers. + */ + if (((*instr >> 24) & 0x1f) == 0b10000) + continue; + + if (pd->kpd_off != -1 && off != pd->kpd_off) + continue; + + /* + * Prevent separate dtrace(1) instances from creating copies of + * the same probe. + */ + LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) { + if (strcmp(kp->kp_func, func) == 0 && + strtol(kp->kp_name, NULL, 10) == off) + return (0); + } + if (++n > KINST_PROBETAB_MAX) { + KINST_LOG("probe list full: %d entries", n); + return (ENOMEM); + } + kp = malloc(sizeof(struct kinst_probe), M_KINST, + M_WAITOK | M_ZERO); + kp->kp_func = func; + snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off); + kp->kp_patchpoint = instr; + kp->kp_savedval = *instr; + kp->kp_patchval = KINST_PATCHVAL; + if ((kp->kp_tramp = kinst_trampoline_alloc(M_WAITOK)) == NULL) { + KINST_LOG("cannot allocate trampoline for %p", instr); + return (ENOMEM); + } + + kinst_instr_dissect(kp); + kinst_probe_create(kp, lf); + } + if (ldxstx_block) + KINST_LOG("warning: unterminated LDX/STX block"); + + return (0); +} + +int +kinst_md_init(void) +{ + struct kinst_cpu_state *ks; + int cpu; + + CPU_FOREACH(cpu) { + ks = DPCPU_PTR(kinst_state); + ks->state = KINST_PROBE_ARMED; + } + + return (0); +} + +void +kinst_md_deinit(void) +{ +} + +/* + * Exclude machine-dependent functions that are not safe-to-trace. + */ +bool +kinst_md_excluded(const char *name) +{ + if (strcmp(name, "handle_el1h_sync") == 0 || + strcmp(name, "do_el1h_sync") == 0) + return (true); + + return (false); +} diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.h b/sys/cddl/dev/kinst/aarch64/kinst_isa.h new file mode 100644 index 000000000000..7e1fd8d123e9 --- /dev/null +++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.h @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Christos Margiolis <christos@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#ifndef _KINST_ISA_H_ +#define _KINST_ISA_H_ + +#define KINST_PATCHVAL DTRACE_PATCHVAL + +/* + * The trampoline contains [instruction, brk]. + */ +#define KINST_TRAMP_SIZE 8 + +typedef uint32_t kinst_patchval_t; + +struct kinst_probe_md { + bool emulate; /* emulate in sw */ +}; + +#endif /* _KINST_ISA_H_ */ diff --git a/sys/cddl/dev/kinst/amd64/kinst_isa.c b/sys/cddl/dev/kinst/amd64/kinst_isa.c new file mode 100644 index 000000000000..b1d3d8727ead --- /dev/null +++ b/sys/cddl/dev/kinst/amd64/kinst_isa.c @@ -0,0 +1,635 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> + * Copyright (c) 2022 Mark Johnston <markj@FreeBSD.org> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> +#include <sys/pcpu.h> + +#include <machine/cpufunc.h> +#include <machine/md_var.h> + +#include <sys/dtrace.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> +#include <dis_tables.h> + +#include "kinst.h" + +#define KINST_PUSHL_RBP 0x55 +#define KINST_STI 0xfb +#define KINST_POPF 0x9d + +#define KINST_MODRM_MOD(b) (((b) & 0xc0) >> 6) +#define KINST_MODRM_REG(b) (((b) & 0x38) >> 3) +#define KINST_MODRM_RM(b) ((b) & 0x07) + +#define KINST_SIB_SCALE(s) (((s) & 0xc0) >> 6) +#define KINST_SIB_INDEX(s) (((s) & 0x38) >> 3) +#define KINST_SIB_BASE(s) (((s) & 0x07) >> 0) + +#define KINST_REX_W(r) (((r) & 0x08) >> 3) +#define KINST_REX_R(r) (((r) & 0x04) >> 2) +#define KINST_REX_X(r) (((r) & 0x02) >> 1) +#define KINST_REX_B(r) (((r) & 0x01) >> 0) + +#define KINST_F_CALL 0x0001 /* instruction is a "call" */ +#define KINST_F_DIRECT_CALL 0x0002 /* instruction is a direct call */ +#define KINST_F_RIPREL 0x0004 /* instruction is position-dependent */ +#define KINST_F_JMP 0x0008 /* instruction is a %rip-relative jmp */ +#define KINST_F_MOD_DIRECT 0x0010 /* operand is not a memory address */ + +/* + * Per-CPU trampolines used when the interrupted thread is executing with + * interrupts disabled. If an interrupt is raised while executing a trampoline, + * the interrupt thread cannot safely overwrite its trampoline if it hits a + * kinst probe while executing the interrupt handler. + */ +DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp); + +/* + * Map ModR/M register bits to a trapframe offset. + */ +static int +kinst_regoff(int reg) +{ +#define _MATCH_REG(i, reg) \ + case i: \ + return (offsetof(struct trapframe, tf_ ## reg) / \ + sizeof(register_t)) + switch (reg) { + _MATCH_REG( 0, rax); + _MATCH_REG( 1, rcx); + _MATCH_REG( 2, rdx); + _MATCH_REG( 3, rbx); + _MATCH_REG( 4, rsp); /* SIB when mod != 3 */ + _MATCH_REG( 5, rbp); + _MATCH_REG( 6, rsi); + _MATCH_REG( 7, rdi); + _MATCH_REG( 8, r8); /* REX.R is set */ + _MATCH_REG( 9, r9); + _MATCH_REG(10, r10); + _MATCH_REG(11, r11); + _MATCH_REG(12, r12); + _MATCH_REG(13, r13); + _MATCH_REG(14, r14); + _MATCH_REG(15, r15); + } +#undef _MATCH_REG + panic("%s: unhandled register index %d", __func__, reg); +} + +/* + * Obtain the specified register's value. + */ +static uint64_t +kinst_regval(struct trapframe *frame, int reg) +{ + if (reg == -1) + return (0); + return (((register_t *)frame)[kinst_regoff(reg)]); +} + +static uint32_t +kinst_riprel_disp(struct kinst_probe *kp, void *dst) +{ + return ((uint32_t)((intptr_t)kp->kp_patchpoint + kp->kp_md.disp - + (intptr_t)dst)); +} + +static void +kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp) +{ + uint8_t *instr; + uint32_t disp; + int ilen; + + ilen = kp->kp_md.tinstlen; + + kinst_memcpy(tramp, kp->kp_md.template, ilen); + if ((kp->kp_md.flags & KINST_F_RIPREL) != 0) { + disp = kinst_riprel_disp(kp, tramp); + kinst_memcpy(&tramp[kp->kp_md.dispoff], &disp, sizeof(uint32_t)); + } + + /* + * The following position-independent jmp takes us back to the + * original code. It is encoded as "jmp *0(%rip)" (six bytes), + * followed by the absolute address of the instruction following + * the one that was traced (eight bytes). + */ + tramp[ilen + 0] = 0xff; + tramp[ilen + 1] = 0x25; + tramp[ilen + 2] = 0x00; + tramp[ilen + 3] = 0x00; + tramp[ilen + 4] = 0x00; + tramp[ilen + 5] = 0x00; + instr = kp->kp_patchpoint + kp->kp_md.instlen; + kinst_memcpy(&tramp[ilen + 6], &instr, sizeof(uintptr_t)); +} + +int +kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) +{ + solaris_cpu_t *cpu; + uintptr_t *stack, retaddr; + struct kinst_probe *kp; + struct kinst_probe_md *kpmd; + uint8_t *tramp; + + stack = (uintptr_t *)frame->tf_rsp; + cpu = &solaris_cpu[curcpu]; + + LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { + if ((uintptr_t)kp->kp_patchpoint == addr) + break; + } + if (kp == NULL) + return (0); + + /* + * Report the address of the breakpoint for the benefit of consumers + * fetching register values with regs[]. + */ + frame->tf_rip--; + + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + cpu->cpu_dtrace_caller = stack[0]; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); + cpu->cpu_dtrace_caller = 0; + + kpmd = &kp->kp_md; + if ((kpmd->flags & KINST_F_CALL) != 0) { + /* + * dtrace_invop_start() reserves space on the stack to + * store the return address of the call instruction. + */ + retaddr = (uintptr_t)(kp->kp_patchpoint + kpmd->instlen); + *(uintptr_t *)scratch = retaddr; + + if ((kpmd->flags & KINST_F_DIRECT_CALL) != 0) { + frame->tf_rip = (uintptr_t)(kp->kp_patchpoint + + kpmd->disp + kpmd->instlen); + } else { + register_t rval; + + if (kpmd->reg1 == -1 && kpmd->reg2 == -1) { + /* rip-relative */ + rval = frame->tf_rip + kpmd->instlen; + } else { + /* indirect */ + rval = kinst_regval(frame, kpmd->reg1) + + (kinst_regval(frame, kpmd->reg2) << + kpmd->scale); + } + + if ((kpmd->flags & KINST_F_MOD_DIRECT) != 0) { + frame->tf_rip = rval + kpmd->disp; + } else { + frame->tf_rip = + *(uintptr_t *)(rval + kpmd->disp); + } + } + return (DTRACE_INVOP_CALL); + } else { + if ((frame->tf_rflags & PSL_I) == 0) + tramp = DPCPU_GET(intr_tramp); + else + tramp = curthread->t_kinst_tramp; + if (tramp == NULL) { + /* + * A trampoline allocation failed, so this probe is + * effectively disabled. Restore the original + * instruction. + * + * We can't safely print anything here, but the + * trampoline allocator should have left a breadcrumb in + * the dmesg. + */ + kinst_patch_tracepoint(kp, kp->kp_savedval); + frame->tf_rip = (register_t)kp->kp_patchpoint; + } else { + kinst_trampoline_populate(kp, tramp); + frame->tf_rip = (register_t)tramp; + } + return (DTRACE_INVOP_NOP); + } +} + +void +kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) +{ + register_t reg; + int oldwp; + + reg = intr_disable(); + oldwp = disable_wp(); + *kp->kp_patchpoint = val; + restore_wp(oldwp); + intr_restore(reg); +} + +static void +kinst_set_disp8(struct kinst_probe *kp, uint8_t byte) +{ + kp->kp_md.disp = (int64_t)(int8_t)byte; +} + +static void +kinst_set_disp32(struct kinst_probe *kp, uint8_t *bytes) +{ + int32_t disp32; + + memcpy(&disp32, bytes, sizeof(disp32)); + kp->kp_md.disp = (int64_t)disp32; +} + +/* + * Set up all of the state needed to faithfully execute a probed instruction. + * + * In the simple case, we copy the instruction unmodified to a per-thread + * trampoline, wherein it is followed by a jump back to the original code. + * - Instructions can have %rip as an operand: + * - with %rip-relative addressing encoded in ModR/M, or + * - implicitly as a part of the instruction definition (jmp, call). + * - Call instructions (which may be %rip-relative) need to push the correct + * return address onto the stack. + * + * Call instructions are simple enough to be emulated in software, so we simply + * do not use the trampoline mechanism in that case. kinst_invop() will compute + * the branch target using the address info computed here (register operands and + * displacement). + * + * %rip-relative operands encoded using the ModR/M byte always use a 32-bit + * displacement; when populating the trampoline the displacement is adjusted to + * be relative to the trampoline address. Trampolines are always allocated + * above KERNBASE for this reason. + * + * For other %rip-relative operands (just jumps) we take the same approach. + * Instructions which specify an 8-bit displacement must be rewritten to use a + * 32-bit displacement. + */ +static int +kinst_instr_dissect(struct kinst_probe *kp, uint8_t **instr) +{ + struct kinst_probe_md *kpmd; + dis86_t d86; + uint8_t *bytes, modrm, rex; + int dispoff, i, ilen, opcidx; + + kpmd = &kp->kp_md; + + d86.d86_data = instr; + d86.d86_get_byte = dtrace_dis_get_byte; + d86.d86_check_func = NULL; + if (dtrace_disx86(&d86, SIZE64) != 0) { + KINST_LOG("failed to disassemble instruction at: %p", *instr); + return (EINVAL); + } + bytes = d86.d86_bytes; + kpmd->instlen = kpmd->tinstlen = d86.d86_len; + + /* + * Skip over prefixes, save REX. + */ + rex = 0; + for (i = 0; i < kpmd->instlen; i++) { + switch (bytes[i]) { + case 0xf0 ... 0xf3: + /* group 1 */ + continue; + case 0x26: + case 0x2e: + case 0x36: + case 0x3e: + case 0x64: + case 0x65: + /* group 2 */ + continue; + case 0x66: + /* group 3 */ + continue; + case 0x67: + /* group 4 */ + continue; + case 0x40 ... 0x4f: + /* REX */ + rex = bytes[i]; + continue; + } + break; + } + KASSERT(i < kpmd->instlen, + ("%s: failed to disassemble instruction at %p", __func__, bytes)); + opcidx = i; + + /* + * Identify instructions of interest by opcode: calls and jumps. + * Extract displacements. + */ + dispoff = -1; + switch (bytes[opcidx]) { + case 0x0f: + switch (bytes[opcidx + 1]) { + case 0x80 ... 0x8f: + /* conditional jmp near */ + kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; + dispoff = opcidx + 2; + kinst_set_disp32(kp, &bytes[dispoff]); + break; + } + break; + case 0xe3: + /* + * There is no straightforward way to translate this instruction + * to use a 32-bit displacement. Fortunately, it is rarely + * used. + */ + return (EINVAL); + case 0x70 ... 0x7f: + /* conditional jmp short */ + kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; + dispoff = opcidx + 1; + kinst_set_disp8(kp, bytes[dispoff]); + break; + case 0xe9: + /* unconditional jmp near */ + kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; + dispoff = opcidx + 1; + kinst_set_disp32(kp, &bytes[dispoff]); + break; + case 0xeb: + /* unconditional jmp short */ + kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; + dispoff = opcidx + 1; + kinst_set_disp8(kp, bytes[dispoff]); + break; + case 0xe8: + case 0x9a: + /* direct call */ + kpmd->flags |= KINST_F_CALL | KINST_F_DIRECT_CALL; + dispoff = opcidx + 1; + kinst_set_disp32(kp, &bytes[dispoff]); + break; + case 0xff: + KASSERT(d86.d86_got_modrm, + ("no ModR/M byte for instr at %p", *instr - kpmd->instlen)); + switch (KINST_MODRM_REG(bytes[d86.d86_rmindex])) { + case 0x02: + case 0x03: + /* indirect call */ + kpmd->flags |= KINST_F_CALL; + break; + case 0x04: + case 0x05: + /* indirect jump */ + kpmd->flags |= KINST_F_JMP; + break; + } + } + + /* + * If there's a ModR/M byte, we need to check it to see if the operand + * is %rip-relative, and rewrite the displacement if so. If not, we + * might still have to extract operand info if this is a call + * instruction. + */ + if (d86.d86_got_modrm) { + uint8_t mod, rm, sib; + + kpmd->reg1 = kpmd->reg2 = -1; + + modrm = bytes[d86.d86_rmindex]; + mod = KINST_MODRM_MOD(modrm); + rm = KINST_MODRM_RM(modrm); + if (mod == 0 && rm == 5) { + kpmd->flags |= KINST_F_RIPREL; + dispoff = d86.d86_rmindex + 1; + kinst_set_disp32(kp, &bytes[dispoff]); + } else if ((kpmd->flags & KINST_F_CALL) != 0) { + bool havesib; + + havesib = (mod != 3 && rm == 4); + dispoff = d86.d86_rmindex + (havesib ? 2 : 1); + if (mod == 1) + kinst_set_disp8(kp, bytes[dispoff]); + else if (mod == 2) + kinst_set_disp32(kp, &bytes[dispoff]); + else if (mod == 3) + kpmd->flags |= KINST_F_MOD_DIRECT; + + if (havesib) { + sib = bytes[d86.d86_rmindex + 1]; + if (KINST_SIB_BASE(sib) != 5) { + kpmd->reg1 = KINST_SIB_BASE(sib) | + (KINST_REX_B(rex) << 3); + } + kpmd->scale = KINST_SIB_SCALE(sib); + kpmd->reg2 = KINST_SIB_INDEX(sib) | + (KINST_REX_X(rex) << 3); + } else { + kpmd->reg1 = rm | (KINST_REX_B(rex) << 3); + } + } + } + + /* + * Calls are emulated in software; once operands are decoded we have + * nothing else to do. + */ + if ((kpmd->flags & KINST_F_CALL) != 0) + return (0); + + /* + * Allocate and populate an instruction trampoline template. + * + * Position-independent instructions can simply be copied, but + * position-dependent instructions require some surgery: jump + * instructions with an 8-bit displacement need to be converted to use a + * 32-bit displacement, and the adjusted displacement needs to be + * computed. + */ + ilen = kpmd->instlen; + if ((kpmd->flags & KINST_F_RIPREL) != 0) { + if ((kpmd->flags & KINST_F_JMP) == 0 || + bytes[opcidx] == 0x0f || + bytes[opcidx] == 0xe9 || + bytes[opcidx] == 0xff) { + memcpy(kpmd->template, bytes, dispoff); + memcpy(&kpmd->template[dispoff + 4], + &bytes[dispoff + 4], ilen - (dispoff + 4)); + kpmd->dispoff = dispoff; + } else if (bytes[opcidx] == 0xeb) { + memcpy(kpmd->template, bytes, opcidx); + kpmd->template[opcidx] = 0xe9; + kpmd->dispoff = opcidx + 1; + + /* Instruction length changes from 2 to 5. */ + kpmd->tinstlen = 5; + kpmd->disp -= 3; + } else if (bytes[opcidx] >= 0x70 && bytes[opcidx] <= 0x7f) { + memcpy(kpmd->template, bytes, opcidx); + kpmd->template[opcidx] = 0x0f; + kpmd->template[opcidx + 1] = bytes[opcidx] + 0x10; + kpmd->dispoff = opcidx + 2; + + /* Instruction length changes from 2 to 6. */ + kpmd->tinstlen = 6; + kpmd->disp -= 4; + } else { + panic("unhandled opcode %#x", bytes[opcidx]); + } + } else { + memcpy(kpmd->template, bytes, ilen); + } + + return (0); +} + +int +kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, + void *opaque) +{ + struct kinst_probe *kp; + dtrace_kinst_probedesc_t *pd; + const char *func; + int error, instrsize, n, off; + uint8_t *instr, *limit, *tmp; + bool push_found; + + pd = opaque; + func = symval->name; + if (kinst_excluded(func)) + return (0); + if (strcmp(func, pd->kpd_func) != 0) + return (0); + + instr = (uint8_t *)symval->value; + limit = (uint8_t *)symval->value + symval->size; + if (instr >= limit) + return (0); + + /* + * Refuse to instrument functions lacking the usual frame pointer + * manipulations since they might correspond to exception handlers. + */ + tmp = instr; + push_found = false; + while (tmp < limit) { + /* + * Checking for 'pop %rbp' as well makes the filtering too + * strict as it would skip functions that never return (e.g., + * vnlru_proc()). + */ + if (*tmp == KINST_PUSHL_RBP) { + push_found = true; + break; + } + tmp += dtrace_instr_size(tmp); + } + if (!push_found) + return (0); + + n = 0; + while (instr < limit) { + instrsize = dtrace_instr_size(instr); + off = (int)(instr - (uint8_t *)symval->value); + if (pd->kpd_off != -1 && off != pd->kpd_off) { + instr += instrsize; + continue; + } + + /* + * Check for instructions which may enable interrupts. Such + * instructions are tricky to trace since it is unclear whether + * to use the per-thread or per-CPU trampolines. Since they are + * rare, we don't bother to implement special handling for them. + * + * If the caller specified an offset, return an error, otherwise + * silently ignore the instruction so that it remains possible + * to enable all instructions in a function. + */ + if (instrsize == 1 && + (instr[0] == KINST_POPF || instr[0] == KINST_STI)) { + if (pd->kpd_off != -1) + return (EINVAL); + instr += instrsize; + continue; + } + + /* + * Prevent separate dtrace(1) instances from creating copies of + * the same probe. + */ + LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) { + if (strcmp(kp->kp_func, func) == 0 && + strtol(kp->kp_name, NULL, 10) == off) + return (0); + } + if (++n > KINST_PROBETAB_MAX) { + KINST_LOG("probe list full: %d entries", n); + return (ENOMEM); + } + kp = malloc(sizeof(struct kinst_probe), M_KINST, + M_WAITOK | M_ZERO); + kp->kp_func = func; + snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off); + kp->kp_savedval = *instr; + kp->kp_patchval = KINST_PATCHVAL; + kp->kp_patchpoint = instr; + + error = kinst_instr_dissect(kp, &instr); + if (error != 0) + return (error); + + kinst_probe_create(kp, lf); + } + + return (0); +} + +int +kinst_md_init(void) +{ + uint8_t *tramp; + int cpu; + + CPU_FOREACH(cpu) { + tramp = kinst_trampoline_alloc(M_WAITOK); + if (tramp == NULL) + return (ENOMEM); + DPCPU_ID_SET(cpu, intr_tramp, tramp); + } + + return (0); +} + +void +kinst_md_deinit(void) +{ + uint8_t *tramp; + int cpu; + + CPU_FOREACH(cpu) { + tramp = DPCPU_ID_GET(cpu, intr_tramp); + if (tramp != NULL) { + kinst_trampoline_dealloc(tramp); + DPCPU_ID_SET(cpu, intr_tramp, NULL); + } + } +} + +/* + * Exclude machine-dependent functions that are not safe-to-trace. + */ +bool +kinst_md_excluded(const char *name) +{ + return (false); +} diff --git a/sys/cddl/dev/kinst/amd64/kinst_isa.h b/sys/cddl/dev/kinst/amd64/kinst_isa.h new file mode 100644 index 000000000000..93d3a1d3ddeb --- /dev/null +++ b/sys/cddl/dev/kinst/amd64/kinst_isa.h @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> + * Copyright (c) 2022 Mark Johnston <markj@FreeBSD.org> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#ifndef _KINST_ISA_H_ +#define _KINST_ISA_H_ + +#include <sys/types.h> + +#define KINST_PATCHVAL 0xcc + +/* + * Each trampoline is 32 bytes long and contains [instruction, jmp]. Since we + * have 2 instructions stored in the trampoline, and each of them can take up + * to 16 bytes, 32 bytes is enough to cover even the worst case scenario. + */ +#define KINST_TRAMP_SIZE 32 + +typedef uint8_t kinst_patchval_t; + +struct kinst_probe_md { + int flags; + int instlen; /* original instr len */ + int tinstlen; /* trampoline instr len */ + uint8_t template[16]; /* copied into thread tramps */ + int dispoff; /* offset of rip displacement */ + + /* operands to "call" instruction branch target */ + int reg1; + int reg2; + int scale; + int64_t disp; +}; + +#endif /* _KINST_ISA_H_ */ diff --git a/sys/cddl/dev/kinst/kinst.c b/sys/cddl/dev/kinst/kinst.c new file mode 100644 index 000000000000..60400a452b95 --- /dev/null +++ b/sys/cddl/dev/kinst/kinst.c @@ -0,0 +1,330 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/conf.h> +#include <sys/kernel.h> +#include <sys/linker.h> +#include <sys/module.h> + +#include <sys/dtrace.h> + +#include "kinst.h" + +MALLOC_DEFINE(M_KINST, "kinst", "Kernel Instruction Tracing"); + +static d_open_t kinst_open; +static d_close_t kinst_close; +static d_ioctl_t kinst_ioctl; + +static void kinst_provide_module(void *, modctl_t *); +static void kinst_getargdesc(void *, dtrace_id_t, void *, + dtrace_argdesc_t *); +static void kinst_destroy(void *, dtrace_id_t, void *); +static void kinst_enable(void *, dtrace_id_t, void *); +static void kinst_disable(void *, dtrace_id_t, void *); +static int kinst_load(void *); +static int kinst_unload(void *); +static int kinst_modevent(module_t, int, void *); + +static dtrace_pattr_t kinst_attr = { +{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, +{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, +}; + +static const dtrace_pops_t kinst_pops = { + .dtps_provide = NULL, + .dtps_provide_module = kinst_provide_module, + .dtps_enable = kinst_enable, + .dtps_disable = kinst_disable, + .dtps_suspend = NULL, + .dtps_resume = NULL, + .dtps_getargdesc = kinst_getargdesc, + .dtps_getargval = NULL, + .dtps_usermode = NULL, + .dtps_destroy = kinst_destroy +}; + +static struct cdevsw kinst_cdevsw = { + .d_name = "kinst", + .d_version = D_VERSION, + .d_flags = D_TRACKCLOSE, + .d_open = kinst_open, + .d_close = kinst_close, + .d_ioctl = kinst_ioctl, +}; + +static dtrace_provider_id_t kinst_id; +struct kinst_probe_list *kinst_probetab; +static struct cdev *kinst_cdev; + +/* + * Tracing memcpy() will crash the kernel when kinst tries to trace an instance + * of the memcpy() calls in kinst_invop(). To fix this, we can use + * kinst_memcpy() in those cases, with its arguments marked as 'volatile' to + * "outsmart" the compiler and avoid having it replaced by a regular memcpy(). + */ +volatile void * +kinst_memcpy(volatile void *dst, volatile const void *src, size_t len) +{ + volatile const unsigned char *src0; + volatile unsigned char *dst0; + + src0 = src; + dst0 = dst; + + while (len--) + *dst0++ = *src0++; + + return (dst); +} + +bool +kinst_excluded(const char *name) +{ + if (kinst_md_excluded(name)) + return (true); + + /* + * cpu_switch() can cause a crash if it modifies the value of curthread + * while in probe context. + */ + if (strcmp(name, "cpu_switch") == 0) + return (true); + + /* + * Anything beginning with "dtrace_" may be called from probe context + * unless it explicitly indicates that it won't be called from probe + * context by using the prefix "dtrace_safe_". + */ + if (strncmp(name, "dtrace_", strlen("dtrace_")) == 0 && + strncmp(name, "dtrace_safe_", strlen("dtrace_safe_")) != 0) + return (true); + + /* + * Omit instrumentation of functions that are probably in DDB. It + * makes it too hard to debug broken kinst. + * + * NB: kdb_enter() can be excluded, but its call to printf() can't be. + * This is generally OK since we're not yet in debugging context. + */ + if (strncmp(name, "db_", strlen("db_")) == 0 || + strncmp(name, "kdb_", strlen("kdb_")) == 0) + return (true); + + /* + * Lock owner methods may be called from probe context. + */ + if (strcmp(name, "owner_mtx") == 0 || + strcmp(name, "owner_rm") == 0 || + strcmp(name, "owner_rw") == 0 || + strcmp(name, "owner_sx") == 0) + return (true); + + /* + * When DTrace is built into the kernel we need to exclude the kinst + * functions from instrumentation. + */ +#ifndef _KLD_MODULE + if (strncmp(name, "kinst_", strlen("kinst_")) == 0) + return (true); +#endif + + if (strcmp(name, "trap_check") == 0) + return (true); + + return (false); +} + +void +kinst_probe_create(struct kinst_probe *kp, linker_file_t lf) +{ + kp->kp_id = dtrace_probe_create(kinst_id, lf->filename, + kp->kp_func, kp->kp_name, 3, kp); + + LIST_INSERT_HEAD(KINST_GETPROBE(kp->kp_patchpoint), kp, kp_hashnext); +} + +static int +kinst_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, + struct thread *td __unused) +{ + return (0); +} + +static int +kinst_close(struct cdev *dev __unused, int fflag __unused, int devtype __unused, + struct thread *td __unused) +{ + dtrace_condense(kinst_id); + return (0); +} + +static int +kinst_linker_file_cb(linker_file_t lf, void *arg) +{ + dtrace_kinst_probedesc_t *pd; + + pd = arg; + if (pd->kpd_mod[0] != '\0' && strcmp(pd->kpd_mod, lf->filename) != 0) + return (0); + + /* + * Invoke kinst_make_probe_function() once for each function symbol in + * the module "lf". + */ + return (linker_file_function_listall(lf, kinst_make_probe, arg)); +} + +static int +kinst_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr, + int flags __unused, struct thread *td __unused) +{ + dtrace_kinst_probedesc_t *pd; + int error = 0; + + switch (cmd) { + case KINSTIOC_MAKEPROBE: + pd = (dtrace_kinst_probedesc_t *)addr; + pd->kpd_func[sizeof(pd->kpd_func) - 1] = '\0'; + pd->kpd_mod[sizeof(pd->kpd_mod) - 1] = '\0'; + + /* Loop over all functions in the kernel and loaded modules. */ + error = linker_file_foreach(kinst_linker_file_cb, pd); + break; + default: + error = ENOTTY; + break; + } + + return (error); +} + +static void +kinst_provide_module(void *arg, modctl_t *lf) +{ +} + +static void +kinst_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) +{ + desc->dtargd_ndx = DTRACE_ARGNONE; +} + +static void +kinst_destroy(void *arg, dtrace_id_t id, void *parg) +{ + struct kinst_probe *kp = parg; + + LIST_REMOVE(kp, kp_hashnext); +#ifndef __amd64__ + kinst_trampoline_dealloc(kp->kp_tramp); +#endif + free(kp, M_KINST); +} + +static void +kinst_enable(void *arg, dtrace_id_t id, void *parg) +{ + struct kinst_probe *kp = parg; + static bool warned = false; + + if (!warned) { + KINST_LOG( + "kinst: This provider is experimental, exercise caution"); + warned = true; + } + + kinst_patch_tracepoint(kp, kp->kp_patchval); +} + +static void +kinst_disable(void *arg, dtrace_id_t id, void *parg) +{ + struct kinst_probe *kp = parg; + + kinst_patch_tracepoint(kp, kp->kp_savedval); +} + +static int +kinst_load(void *dummy) +{ + int error; + + error = kinst_trampoline_init(); + if (error != 0) + return (error); + error = kinst_md_init(); + if (error != 0) { + kinst_trampoline_deinit(); + return (error); + } + + error = dtrace_register("kinst", &kinst_attr, DTRACE_PRIV_USER, NULL, + &kinst_pops, NULL, &kinst_id); + if (error != 0) { + kinst_md_deinit(); + kinst_trampoline_deinit(); + return (error); + } + kinst_probetab = malloc(KINST_PROBETAB_MAX * + sizeof(struct kinst_probe_list), M_KINST, M_WAITOK | M_ZERO); + for (int i = 0; i < KINST_PROBETAB_MAX; i++) + LIST_INIT(&kinst_probetab[i]); + kinst_cdev = make_dev(&kinst_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, + "dtrace/kinst"); + dtrace_invop_add(kinst_invop); + return (0); +} + +static int +kinst_unload(void *dummy) +{ + free(kinst_probetab, M_KINST); + kinst_md_deinit(); + kinst_trampoline_deinit(); + dtrace_invop_remove(kinst_invop); + destroy_dev(kinst_cdev); + + return (dtrace_unregister(kinst_id)); +} + +static int +kinst_modevent(module_t mod __unused, int type, void *data __unused) +{ + int error = 0; + + switch (type) { + case MOD_LOAD: + break; + case MOD_UNLOAD: + break; + case MOD_SHUTDOWN: + break; + default: + error = EOPNOTSUPP; + break; + } + + return (error); +} + +SYSINIT(kinst_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_load, NULL); +SYSUNINIT(kinst_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_unload, + NULL); + +DEV_MODULE(kinst, kinst_modevent, NULL); +MODULE_VERSION(kinst, 1); +MODULE_DEPEND(kinst, dtrace, 1, 1, 1); +MODULE_DEPEND(kinst, opensolaris, 1, 1, 1); diff --git a/sys/cddl/dev/kinst/kinst.h b/sys/cddl/dev/kinst/kinst.h new file mode 100644 index 000000000000..f0811a4a88d8 --- /dev/null +++ b/sys/cddl/dev/kinst/kinst.h @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#ifndef _KINST_H_ +#define _KINST_H_ + +#include <sys/dtrace.h> + +typedef struct { + char kpd_func[DTRACE_FUNCNAMELEN]; + char kpd_mod[DTRACE_MODNAMELEN]; + int kpd_off; +} dtrace_kinst_probedesc_t; + +#define KINSTIOC_MAKEPROBE _IOW('k', 1, dtrace_kinst_probedesc_t) + +#ifdef _KERNEL + +#include <sys/queue.h> + +#include "kinst_isa.h" + +struct kinst_probe { + LIST_ENTRY(kinst_probe) kp_hashnext; + const char *kp_func; + char kp_name[16]; + dtrace_id_t kp_id; + kinst_patchval_t kp_patchval; + kinst_patchval_t kp_savedval; + kinst_patchval_t *kp_patchpoint; + uint8_t *kp_tramp; + + struct kinst_probe_md kp_md; +}; + +struct kinst_cpu_state { + /* + * kinst uses a breakpoint to return from the trampoline and resume + * execution. To do this safely, kinst implements a per-CPU state + * machine; the state is set to KINST_PROBE_FIRED for the duration of + * the trampoline execution (i.e from the time we transfer execution to + * it, until we return). Upon return, the state is set to + * KINST_PROBE_ARMED to indicate that a probe is not currently firing. + * All CPUs have their state initialized to KINST_PROBE_ARMED when + * kinst is loaded. + */ + enum { + KINST_PROBE_ARMED, + KINST_PROBE_FIRED, + } state; + /* + * Points to the probe whose trampoline we're currently executing. + */ + const struct kinst_probe *kp; + /* + * Because we execute trampolines with interrupts disabled, we have to + * cache the CPU's status in order to restore it when we return from + * the trampoline. + */ + uint64_t status; +}; + +LIST_HEAD(kinst_probe_list, kinst_probe); + +extern struct kinst_probe_list *kinst_probetab; + +#define KINST_PROBETAB_MAX 0x8000 /* 32k */ +#define KINST_ADDR2NDX(addr) (((uintptr_t)(addr)) & (KINST_PROBETAB_MAX - 1)) +#define KINST_GETPROBE(i) (&kinst_probetab[KINST_ADDR2NDX(i)]) + +struct linker_file; +struct linker_symval; + +/* kinst.c */ +volatile void *kinst_memcpy(volatile void *, volatile const void *, size_t); +bool kinst_excluded(const char *); +void kinst_probe_create(struct kinst_probe *, struct linker_file *); + +/* arch/kinst_isa.c */ +int kinst_invop(uintptr_t, struct trapframe *, uintptr_t); +void kinst_patch_tracepoint(struct kinst_probe *, kinst_patchval_t); +int kinst_make_probe(struct linker_file *, int, struct linker_symval *, + void *); +int kinst_md_init(void); +void kinst_md_deinit(void); +bool kinst_md_excluded(const char *); + +/* trampoline.c */ +int kinst_trampoline_init(void); +int kinst_trampoline_deinit(void); +uint8_t *kinst_trampoline_alloc(int); +void kinst_trampoline_dealloc(uint8_t *); + +#ifdef MALLOC_DECLARE +MALLOC_DECLARE(M_KINST); +#endif /* MALLOC_DECLARE */ + +#define KINST_LOG_HELPER(fmt, ...) \ + printf("%s:%d: " fmt "%s\n", __func__, __LINE__, __VA_ARGS__) +#define KINST_LOG(...) \ + KINST_LOG_HELPER(__VA_ARGS__, "") + +#endif /* _KERNEL */ + +#endif /* _KINST_H_ */ diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.c b/sys/cddl/dev/kinst/riscv/kinst_isa.c new file mode 100644 index 000000000000..1fabde189712 --- /dev/null +++ b/sys/cddl/dev/kinst/riscv/kinst_isa.c @@ -0,0 +1,586 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Christos Margiolis <christos@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> + +#include <sys/dtrace.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> + +#include "kinst.h" + +DPCPU_DEFINE_STATIC(struct kinst_cpu_state, kinst_state); + +#define _MATCH_REG(reg) \ + (offsetof(struct trapframe, tf_ ## reg) / sizeof(register_t)) + +static int +kinst_regoff(struct trapframe *frame, int n) +{ + switch (n) { + case 0: + /* There is no zero register in the trapframe structure. */ + return (-1); + case 1: + return (_MATCH_REG(ra)); + case 2: + return (_MATCH_REG(sp)); + case 3: + return (_MATCH_REG(gp)); + case 4: + return (_MATCH_REG(tp)); + case 5 ... 7: + return (_MATCH_REG(t[n - 5])); + case 8 ... 9: + return (_MATCH_REG(s[n - 8])); + case 10 ... 17: + return (_MATCH_REG(a[n - 10])); + case 18 ... 27: + return (_MATCH_REG(s[n - 18 + 2])); + case 28 ... 31: + return (_MATCH_REG(t[n - 28 + 3])); + default: + panic("%s: unhandled register index %d", __func__, n); + } +} + +static int +kinst_c_regoff(struct trapframe *frame, int n) +{ + switch (n) { + case 0 ... 1: + return (_MATCH_REG(s[n])); + case 2 ... 7: + return (_MATCH_REG(a[n - 2])); + default: + panic("%s: unhandled register index %d", __func__, n); + } +} + +#undef _MATCH_REG + +static int +kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp) +{ + kinst_patchval_t instr = kp->kp_savedval; + register_t prevpc; + uint64_t imm; + uint16_t off; + uint8_t funct; + + if (kp->kp_md.instlen == INSN_SIZE) { +#define rs1_index ((instr & RS1_MASK) >> RS1_SHIFT) +#define rs2_index ((instr & RS2_MASK) >> RS2_SHIFT) +#define rd_index ((instr & RD_MASK) >> RD_SHIFT) +#define rs1 ((register_t *)frame)[kinst_regoff(frame, rs1_index)] +#define rs2 ((register_t *)frame)[kinst_regoff(frame, rs2_index)] +#define rd ((register_t *)frame)[kinst_regoff(frame, rd_index)] +#define rs1_lval (rs1_index != 0 ? rs1 : 0) +#define rs2_lval (rs2_index != 0 ? rs2 : 0) + switch (instr & 0x7f) { + case 0b1101111: /* jal */ + imm = 0; + imm |= ((instr >> 21) & 0x03ff) << 1; + imm |= ((instr >> 20) & 0x0001) << 11; + imm |= ((instr >> 12) & 0x00ff) << 12; + imm |= ((instr >> 31) & 0x0001) << 20; + if (imm & 0x0000000000100000) + imm |= 0xfffffffffff00000; + if (rd_index != 0) + rd = frame->tf_sepc + INSN_SIZE; + frame->tf_sepc += imm; + break; + case 0b1100111: /* jalr */ + prevpc = frame->tf_sepc; + imm = (instr & IMM_MASK) >> IMM_SHIFT; + if (imm & 0x0000000000000800) + imm |= 0xfffffffffffff000; + frame->tf_sepc = (rs1_lval + imm) & ~1; + if (rd_index != 0) + rd = prevpc + INSN_SIZE; + break; + case 0b1100011: /* branch */ + imm = 0; + imm |= ((instr >> 8) & 0x000f) << 1; + imm |= ((instr >> 25) & 0x003f) << 5; + imm |= ((instr >> 7) & 0x0001) << 11; + imm |= ((instr >> 31) & 0x0001) << 12; + if (imm & 0x0000000000001000) + imm |= 0xfffffffffffff000; + funct = (instr >> 12) & 0x07; + switch (funct) { + case 0b000: /* beq */ + if (rs1_lval == rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b001: /* bne */ + if (rs1_lval != rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b100: /* blt */ + if ((int64_t)rs1_lval < (int64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b110: /* bltu */ + if ((uint64_t)rs1_lval < (uint64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b101: /* bge */ + if ((int64_t)rs1_lval >= (int64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b111: /* bgeu */ + if ((uint64_t)rs1_lval >= (uint64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + } + break; + case 0b0010111: /* auipc */ + imm = instr & 0xfffff000; + rd = frame->tf_sepc + + (imm & 0x0000000080000000 ? + imm | 0xffffffff80000000 : imm); + frame->tf_sepc += INSN_SIZE; + break; + } +#undef rs1_lval +#undef rs2_lval +#undef rs1 +#undef rs2 +#undef rd +#undef rs1_index +#undef rs2_index +#undef rd_index + } else { + switch (instr & 0x03) { +#define rs1 \ + ((register_t *)frame)[kinst_c_regoff(frame, (instr >> 7) & 0x07)] + case 0b01: + funct = (instr >> 13) & 0x07; + switch (funct) { + case 0b101: /* c.j */ + off = (instr >> 2) & 0x07ff; + imm = 0; + imm |= ((off >> 1) & 0x07) << 1; + imm |= ((off >> 9) & 0x01) << 4; + imm |= ((off >> 0) & 0x01) << 5; + imm |= ((off >> 5) & 0x01) << 6; + imm |= ((off >> 4) & 0x01) << 7; + imm |= ((off >> 7) & 0x03) << 8; + imm |= ((off >> 6) & 0x01) << 10; + imm |= ((off >> 10) & 0x01) << 11; + if (imm & 0x0000000000000800) + imm |= 0xfffffffffffff000; + frame->tf_sepc += imm; + break; + case 0b110: /* c.beqz */ + case 0b111: /* c.bnez */ + imm = 0; + imm |= ((instr >> 3) & 0x03) << 1; + imm |= ((instr >> 10) & 0x03) << 3; + imm |= ((instr >> 2) & 0x01) << 5; + imm |= ((instr >> 5) & 0x03) << 6; + imm |= ((instr >> 12) & 0x01) << 8; + if (imm & 0x0000000000000100) + imm |= 0xffffffffffffff00; + if (funct == 0b110 && rs1 == 0) + frame->tf_sepc += imm; + else if (funct == 0b111 && rs1 != 0) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_C_SIZE; + break; + } + break; +#undef rs1 +#define rs1_index ((instr & RD_MASK) >> RD_SHIFT) +#define rs1 ((register_t *)frame)[kinst_regoff(frame, rs1_index)] + case 0b10: + funct = (instr >> 13) & 0x07; + if (funct == 0b100 && rs1_index != 0) { + /* c.jr/c.jalr */ + prevpc = frame->tf_sepc; + frame->tf_sepc = rs1; + if (((instr >> 12) & 0x01) != 0) + frame->tf_ra = prevpc + INSN_C_SIZE; + } + break; +#undef rs1 +#undef rs1_index + } + } + + return (MATCH_C_NOP); +} + +static int +kinst_jump_next_instr(struct trapframe *frame, const struct kinst_probe *kp) +{ + frame->tf_sepc = (register_t)((const uint8_t *)kp->kp_patchpoint + + kp->kp_md.instlen); + + return (MATCH_C_NOP); +} + +static void +kinst_trampoline_populate(struct kinst_probe *kp) +{ + static uint16_t nop = MATCH_C_NOP; + static uint32_t ebreak = MATCH_EBREAK; + int ilen; + + ilen = kp->kp_md.instlen; + kinst_memcpy(kp->kp_tramp, &kp->kp_savedval, ilen); + + /* + * Since we cannot encode large displacements in a single instruction + * in order to encode a far-jump back to the next instruction, and we + * also cannot clobber a register inside the trampoline, we execute a + * breakpoint after the copied instruction. kinst_invop() is + * responsible for detecting this special case and performing the + * "jump" manually. + * + * Add a NOP after a compressed instruction for padding. + */ + if (ilen == INSN_C_SIZE) + kinst_memcpy(&kp->kp_tramp[ilen], &nop, INSN_C_SIZE); + + kinst_memcpy(&kp->kp_tramp[INSN_SIZE], &ebreak, INSN_SIZE); + + fence_i(); +} + +/* + * There are two ways by which an instruction is traced: + * + * - By using the trampoline. + * - By emulating it in software (see kinst_emulate()). + * + * The trampoline is used for instructions that can be copied and executed + * as-is without additional modification. However, instructions that use + * PC-relative addressing have to be emulated, because RISC-V doesn't allow + * encoding of large displacements in a single instruction, and since we cannot + * clobber a register in order to encode the two-instruction sequence needed to + * create large displacements, we cannot use the trampoline at all. + * Fortunately, the instructions are simple enough to be emulated in just a few + * lines of code. + * + * The problem discussed above also means that, unlike amd64, we cannot encode + * a far-jump back from the trampoline to the next instruction. The mechanism + * employed to achieve this functionality, is to use a breakpoint instead of a + * jump after the copied instruction. This breakpoint is detected and handled + * by kinst_invop(), which performs the jump back to the next instruction + * manually (see kinst_jump_next_instr()). + */ +int +kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) +{ + solaris_cpu_t *cpu; + struct kinst_cpu_state *ks; + const struct kinst_probe *kp; + + ks = DPCPU_PTR(kinst_state); + + /* + * Detect if the breakpoint was triggered by the trampoline, and + * manually set the PC to the next instruction. + */ + if (ks->state == KINST_PROBE_FIRED && + addr == (uintptr_t)(ks->kp->kp_tramp + INSN_SIZE)) { + /* + * Restore interrupts if they were enabled prior to the first + * breakpoint. + */ + if ((ks->status & SSTATUS_SPIE) != 0) + frame->tf_sstatus |= SSTATUS_SPIE; + ks->state = KINST_PROBE_ARMED; + return (kinst_jump_next_instr(frame, ks->kp)); + } + + LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { + if ((uintptr_t)kp->kp_patchpoint == addr) + break; + } + if (kp == NULL) + return (0); + + cpu = &solaris_cpu[curcpu]; + cpu->cpu_dtrace_caller = addr; + dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); + cpu->cpu_dtrace_caller = 0; + + if (kp->kp_md.emulate) + return (kinst_emulate(frame, kp)); + + ks->state = KINST_PROBE_FIRED; + ks->kp = kp; + + /* + * Cache the current SSTATUS and clear interrupts for the + * duration of the double breakpoint. + */ + ks->status = frame->tf_sstatus; + frame->tf_sstatus &= ~SSTATUS_SPIE; + frame->tf_sepc = (register_t)kp->kp_tramp; + + return (MATCH_C_NOP); +} + +void +kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) +{ + switch (kp->kp_patchval) { + case KINST_C_PATCHVAL: + *(uint16_t *)kp->kp_patchpoint = (uint16_t)val; + fence_i(); + break; + case KINST_PATCHVAL: + *kp->kp_patchpoint = val; + fence_i(); + break; + } +} + +static void +kinst_instr_dissect(struct kinst_probe *kp, int instrsize) +{ + struct kinst_probe_md *kpmd; + kinst_patchval_t instr = kp->kp_savedval; + uint8_t funct; + + kpmd = &kp->kp_md; + kpmd->instlen = instrsize; + kpmd->emulate = false; + + /* + * The following instructions use PC-relative addressing and need to be + * emulated in software. + */ + if (kpmd->instlen == INSN_SIZE) { + switch (instr & 0x7f) { + case 0b1101111: /* jal */ + case 0b1100111: /* jalr */ + case 0b1100011: /* branch */ + case 0b0010111: /* auipc */ + kpmd->emulate = true; + break; + } + } else { + switch (instr & 0x03) { + case 0b01: + funct = (instr >> 13) & 0x07; + switch (funct) { + case 0b101: /* c.j */ + case 0b110: /* c.beqz */ + case 0b111: /* c.bnez */ + kpmd->emulate = true; + break; + } + break; + case 0b10: + funct = (instr >> 13) & 0x07; + if (funct == 0b100 && + ((instr >> 7) & 0x1f) != 0 && + ((instr >> 2) & 0x1f) == 0) + kpmd->emulate = true; /* c.jr/c.jalr */ + break; + } + } + + if (!kpmd->emulate) + kinst_trampoline_populate(kp); +} + +static bool +kinst_instr_system(kinst_patchval_t instr) +{ + if (dtrace_match_opcode(instr, MATCH_C_EBREAK, MASK_C_EBREAK) || + (instr & 0x7f) == 0b1110011) + return (true); + + return (false); +} + +static bool +kinst_instr_lr(kinst_patchval_t instr) +{ + if (dtrace_match_opcode(instr, MATCH_LR_W, MASK_LR_W) || + dtrace_match_opcode(instr, MATCH_LR_D, MASK_LR_D)) + return (true); + + return (false); +} + +static bool +kinst_instr_sc(kinst_patchval_t instr) +{ + if (dtrace_match_opcode(instr, MATCH_SC_W, MASK_SC_W) || + dtrace_match_opcode(instr, MATCH_SC_D, MASK_SC_D)) + return (true); + + return (false); +} + +int +kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, + void *opaque) +{ + struct kinst_probe *kp; + dtrace_kinst_probedesc_t *pd; + const char *func; + kinst_patchval_t *insn, v; + uint8_t *instr, *limit; + int instrsize, n, off; + bool lrsc_block, store_found; + + pd = opaque; + func = symval->name; + + if (kinst_excluded(func)) + return (0); + if (strcmp(func, pd->kpd_func) != 0) + return (0); + + instr = (uint8_t *)(symval->value); + limit = (uint8_t *)(symval->value + symval->size); + if (instr >= limit) + return (0); + + /* Check for the usual function prologue. */ + store_found = false; + for (insn = (kinst_patchval_t *)instr; + insn < (kinst_patchval_t *)limit; insn++) { + if (dtrace_instr_sdsp(&insn) || dtrace_instr_c_sdsp(&insn)) { + store_found = true; + break; + } + } + if (!store_found) + return (0); + + n = 0; + lrsc_block = false; + while (instr < limit) { + instrsize = dtrace_instr_size(instr); + off = (int)(instr - (uint8_t *)symval->value); + + /* + * Avoid undefined behavior (i.e simply casting `*instr` to + * `kinst_patchval_t`) in case the pointer is unaligned. + * memcpy() can safely operate on unaligned pointers. + */ + memcpy(&v, instr, sizeof(kinst_patchval_t)); + + /* Skip SYSTEM instructions. */ + if (kinst_instr_system(v)) + goto cont; + + /* + * Skip LR/SC blocks used to build atomic operations. If a + * breakpoint is placed in a LR/SC block, the loop becomes + * unconstrained. In this case we violate the operation and the + * loop might fail on some implementations (see section 8.3 of + * the RISC-V unprivileged spec). + */ + if (kinst_instr_lr(v)) + lrsc_block = true; + else if (kinst_instr_sc(v)) { + lrsc_block = false; + goto cont; + } + if (lrsc_block) + goto cont; + + if (pd->kpd_off != -1 && off != pd->kpd_off) + goto cont; + + /* + * Prevent separate dtrace(1) instances from creating copies of + * the same probe. + */ + LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) { + if (strcmp(kp->kp_func, func) == 0 && + strtol(kp->kp_name, NULL, 10) == off) + return (0); + } + if (++n > KINST_PROBETAB_MAX) { + KINST_LOG("probe list full: %d entries", n); + return (ENOMEM); + } + kp = malloc(sizeof(struct kinst_probe), M_KINST, + M_WAITOK | M_ZERO); + kp->kp_func = func; + snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off); + kp->kp_patchpoint = (kinst_patchval_t *)instr; + kp->kp_savedval = v; + if (instrsize == INSN_SIZE) + kp->kp_patchval = KINST_PATCHVAL; + else + kp->kp_patchval = KINST_C_PATCHVAL; + if ((kp->kp_tramp = kinst_trampoline_alloc(M_WAITOK)) == NULL) { + KINST_LOG("cannot allocate trampoline for %p", instr); + return (ENOMEM); + } + + kinst_instr_dissect(kp, instrsize); + kinst_probe_create(kp, lf); +cont: + instr += instrsize; + } + if (lrsc_block) + KINST_LOG("warning: unterminated LR/SC block"); + + return (0); +} + +int +kinst_md_init(void) +{ + struct kinst_cpu_state *ks; + int cpu; + + CPU_FOREACH(cpu) { + ks = DPCPU_PTR(kinst_state); + ks->state = KINST_PROBE_ARMED; + } + + return (0); +} + +void +kinst_md_deinit(void) +{ +} + +/* + * Exclude machine-dependent functions that are not safe-to-trace. + */ +bool +kinst_md_excluded(const char *name) +{ + if (strcmp(name, "cpu_exception_handler") == 0 || + strcmp(name, "cpu_exception_handler_supervisor") == 0 || + strcmp(name, "cpu_exception_handler_user") == 0 || + strcmp(name, "do_trap_supervisor") == 0 || + strcmp(name, "do_trap_user") == 0) + return (true); + + return (false); +} diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.h b/sys/cddl/dev/kinst/riscv/kinst_isa.h new file mode 100644 index 000000000000..bd7c2b3b8dcf --- /dev/null +++ b/sys/cddl/dev/kinst/riscv/kinst_isa.h @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Christos Margiolis <christos@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#ifndef _KINST_ISA_H_ +#define _KINST_ISA_H_ + +#include <machine/riscvreg.h> +#include <machine/encoding.h> + +#define KINST_PATCHVAL MATCH_EBREAK +#define KINST_C_PATCHVAL MATCH_C_EBREAK + +/* + * The trampoline contains [instruction, [nop padding], ebreak]. + */ +#define KINST_TRAMP_SIZE 8 + +typedef uint32_t kinst_patchval_t; + +struct kinst_probe_md { + int instlen; /* original instr len */ + bool emulate; /* emulate in sw */ +}; + +#endif /* _KINST_ISA_H_ */ diff --git a/sys/cddl/dev/kinst/trampoline.c b/sys/cddl/dev/kinst/trampoline.c new file mode 100644 index 000000000000..adc4eaa7fceb --- /dev/null +++ b/sys/cddl/dev/kinst/trampoline.c @@ -0,0 +1,354 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org> + * Copyright (c) 2022 Mark Johnston <markj@FreeBSD.org> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> +#include <sys/bitset.h> +#include <sys/cred.h> +#include <sys/eventhandler.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/proc.h> +#include <sys/queue.h> +#include <sys/sx.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <vm/vm_kern.h> +#include <vm/vm_object.h> + +#include <cddl/dev/dtrace/dtrace_cddl.h> + +#include "kinst.h" +#include "kinst_isa.h" + +#define KINST_TRAMP_FILL_PATTERN ((kinst_patchval_t []){KINST_PATCHVAL}) +#define KINST_TRAMP_FILL_SIZE sizeof(kinst_patchval_t) + +#define KINST_TRAMPCHUNK_SIZE PAGE_SIZE +#define KINST_TRAMPS_PER_CHUNK (KINST_TRAMPCHUNK_SIZE / KINST_TRAMP_SIZE) + +struct trampchunk { + TAILQ_ENTRY(trampchunk) next; + uint8_t *addr; + /* 0 -> allocated, 1 -> free */ + BITSET_DEFINE(, KINST_TRAMPS_PER_CHUNK) free; +}; + +static TAILQ_HEAD(, trampchunk) kinst_trampchunks = + TAILQ_HEAD_INITIALIZER(kinst_trampchunks); +static struct sx kinst_tramp_sx; +SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp"); +#ifdef __amd64__ +static eventhandler_tag kinst_thread_ctor_handler; +static eventhandler_tag kinst_thread_dtor_handler; +#endif + +/* + * Fill the trampolines with KINST_TRAMP_FILL_PATTERN so that the kernel will + * crash cleanly if things somehow go wrong. + */ +static void +kinst_trampoline_fill(uint8_t *addr, int size) +{ + int i; + + for (i = 0; i < size; i += KINST_TRAMP_FILL_SIZE) { + memcpy(&addr[i], KINST_TRAMP_FILL_PATTERN, + KINST_TRAMP_FILL_SIZE); + } +} + +static struct trampchunk * +kinst_trampchunk_alloc(void) +{ + struct trampchunk *chunk; + vm_offset_t trampaddr; + int error __diagused; + + sx_assert(&kinst_tramp_sx, SX_XLOCKED); + +#ifdef __amd64__ + /* + * To simplify population of trampolines, we follow the amd64 kernel's + * code model and allocate them above KERNBASE, i.e., in the top 2GB of + * the kernel's virtual address space (not the case for other + * platforms). + */ + trampaddr = KERNBASE; +#else + trampaddr = VM_MIN_KERNEL_ADDRESS; +#endif + /* + * Allocate virtual memory for the trampoline chunk. The returned + * address is saved in "trampaddr". Trampolines must be executable so + * max_prot must include VM_PROT_EXECUTE. + */ + error = vm_map_find(kernel_map, NULL, 0, &trampaddr, + KINST_TRAMPCHUNK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, + 0); + if (error != KERN_SUCCESS) { + KINST_LOG("trampoline chunk allocation failed: %d", error); + return (NULL); + } + + error = kmem_back(kernel_object, trampaddr, KINST_TRAMPCHUNK_SIZE, + M_WAITOK | M_EXEC); + KASSERT(error == KERN_SUCCESS, ("kmem_back failed: %d", error)); + + kinst_trampoline_fill((uint8_t *)trampaddr, KINST_TRAMPCHUNK_SIZE); + + /* Allocate a tracker for this chunk. */ + chunk = malloc(sizeof(*chunk), M_KINST, M_WAITOK); + chunk->addr = (void *)trampaddr; + BIT_FILL(KINST_TRAMPS_PER_CHUNK, &chunk->free); + + TAILQ_INSERT_HEAD(&kinst_trampchunks, chunk, next); + + return (chunk); +} + +static void +kinst_trampchunk_free(struct trampchunk *chunk) +{ + sx_assert(&kinst_tramp_sx, SX_XLOCKED); + + TAILQ_REMOVE(&kinst_trampchunks, chunk, next); + kmem_unback(kernel_object, (vm_offset_t)chunk->addr, + KINST_TRAMPCHUNK_SIZE); + (void)vm_map_remove(kernel_map, (vm_offset_t)chunk->addr, + (vm_offset_t)(chunk->addr + KINST_TRAMPCHUNK_SIZE)); + free(chunk, M_KINST); +} + +static uint8_t * +kinst_trampoline_alloc_locked(int how) +{ + struct trampchunk *chunk; + uint8_t *tramp; + int off; + + sx_assert(&kinst_tramp_sx, SX_XLOCKED); + + TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { + /* All trampolines from this chunk are already allocated. */ + if ((off = BIT_FFS(KINST_TRAMPS_PER_CHUNK, &chunk->free)) == 0) + continue; + /* BIT_FFS() returns indices starting at 1 instead of 0. */ + off--; + break; + } + if (chunk == NULL) { + if ((how & M_NOWAIT) != 0) + return (NULL); + + if ((chunk = kinst_trampchunk_alloc()) == NULL) { +#ifdef __amd64__ + /* + * We didn't find any free trampoline in the current + * list, allocate a new one. If that fails the + * provider will no longer be reliable, so try to warn + * the user. + */ + static bool once = true; + + if (once) { + once = false; + KINST_LOG( + "kinst: failed to allocate trampoline, " + "probes may not fire"); + } +#endif + return (NULL); + } + off = 0; + } + BIT_CLR(KINST_TRAMPS_PER_CHUNK, off, &chunk->free); + tramp = chunk->addr + off * KINST_TRAMP_SIZE; + return (tramp); +} + +uint8_t * +kinst_trampoline_alloc(int how) +{ + uint8_t *tramp; + + sx_xlock(&kinst_tramp_sx); + tramp = kinst_trampoline_alloc_locked(how); + sx_xunlock(&kinst_tramp_sx); + return (tramp); +} + +static void +kinst_trampoline_dealloc_locked(uint8_t *tramp, bool freechunks) +{ + struct trampchunk *chunk; + int off; + + sx_assert(&kinst_tramp_sx, SX_XLOCKED); + + if (tramp == NULL) + return; + + TAILQ_FOREACH(chunk, &kinst_trampchunks, next) { + for (off = 0; off < KINST_TRAMPS_PER_CHUNK; off++) { + if (chunk->addr + off * KINST_TRAMP_SIZE == tramp) { + kinst_trampoline_fill(tramp, KINST_TRAMP_SIZE); + BIT_SET(KINST_TRAMPS_PER_CHUNK, off, + &chunk->free); + if (freechunks && + BIT_ISFULLSET(KINST_TRAMPS_PER_CHUNK, + &chunk->free)) + kinst_trampchunk_free(chunk); + return; + } + } + } + panic("%s: did not find trampoline chunk for %p", __func__, tramp); +} + +void +kinst_trampoline_dealloc(uint8_t *tramp) +{ + sx_xlock(&kinst_tramp_sx); + kinst_trampoline_dealloc_locked(tramp, true); + sx_xunlock(&kinst_tramp_sx); +} + +#ifdef __amd64__ +static void +kinst_thread_ctor(void *arg __unused, struct thread *td) +{ + td->t_kinst_tramp = kinst_trampoline_alloc(M_WAITOK); +} + +static void +kinst_thread_dtor(void *arg __unused, struct thread *td) +{ + void *tramp; + + tramp = td->t_kinst_tramp; + td->t_kinst_tramp = NULL; + + /* + * This assumes that the thread_dtor event permits sleeping, which + * appears to be true for the time being. + */ + kinst_trampoline_dealloc(tramp); +} +#endif + +int +kinst_trampoline_init(void) +{ +#ifdef __amd64__ + struct proc *p; + struct thread *td; + void *tramp; + int error; + + kinst_thread_ctor_handler = EVENTHANDLER_REGISTER(thread_ctor, + kinst_thread_ctor, NULL, EVENTHANDLER_PRI_ANY); + kinst_thread_dtor_handler = EVENTHANDLER_REGISTER(thread_dtor, + kinst_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); + + error = 0; + tramp = NULL; + + sx_slock(&allproc_lock); + sx_xlock(&kinst_tramp_sx); + FOREACH_PROC_IN_SYSTEM(p) { +retry: + PROC_LOCK(p); + FOREACH_THREAD_IN_PROC(p, td) { + if (td->t_kinst_tramp != NULL) + continue; + if (tramp == NULL) { + /* + * Try to allocate a trampoline without dropping + * the process lock. If all chunks are fully + * utilized, we must release the lock and try + * again. + */ + tramp = kinst_trampoline_alloc_locked(M_NOWAIT); + if (tramp == NULL) { + PROC_UNLOCK(p); + tramp = kinst_trampoline_alloc_locked( + M_WAITOK); + if (tramp == NULL) { + /* + * Let the unload handler clean + * up. + */ + error = ENOMEM; + goto out; + } else + goto retry; + } + } + td->t_kinst_tramp = tramp; + tramp = NULL; + } + PROC_UNLOCK(p); + } +out: + sx_xunlock(&kinst_tramp_sx); + sx_sunlock(&allproc_lock); +#else + int error = 0; + + sx_xlock(&kinst_tramp_sx); + TAILQ_INIT(&kinst_trampchunks); + sx_xunlock(&kinst_tramp_sx); +#endif + + return (error); +} + +int +kinst_trampoline_deinit(void) +{ +#ifdef __amd64__ + struct trampchunk *chunk, *tmp; + struct proc *p; + struct thread *td; + + EVENTHANDLER_DEREGISTER(thread_ctor, kinst_thread_ctor_handler); + EVENTHANDLER_DEREGISTER(thread_dtor, kinst_thread_dtor_handler); + + sx_slock(&allproc_lock); + sx_xlock(&kinst_tramp_sx); + FOREACH_PROC_IN_SYSTEM(p) { + PROC_LOCK(p); + FOREACH_THREAD_IN_PROC(p, td) { + kinst_trampoline_dealloc_locked(td->t_kinst_tramp, + false); + td->t_kinst_tramp = NULL; + } + PROC_UNLOCK(p); + } + sx_sunlock(&allproc_lock); + TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) + kinst_trampchunk_free(chunk); + sx_xunlock(&kinst_tramp_sx); +#else + struct trampchunk *chunk, *tmp; + + sx_xlock(&kinst_tramp_sx); + TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) + kinst_trampchunk_free(chunk); + sx_xunlock(&kinst_tramp_sx); +#endif + + return (0); +} diff --git a/sys/cddl/dev/profile/profile.c b/sys/cddl/dev/profile/profile.c index 89b0b8c9445b..677c7543795f 100644 --- a/sys/cddl/dev/profile/profile.c +++ b/sys/cddl/dev/profile/profile.c @@ -20,8 +20,6 @@ * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * - * $FreeBSD$ - * */ /* @@ -29,7 +27,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/conf.h> @@ -60,6 +57,8 @@ #include <sys/dtrace.h> #include <sys/dtrace_bsd.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> + #define PROF_NAMELEN 15 #define PROF_PROFILE 0 @@ -97,27 +96,15 @@ #endif #endif -#ifdef __mips -/* - * This value is bogus just to make module compilable on mips - */ -#define PROF_ARTIFICIAL_FRAMES 3 -#endif - #ifdef __powerpc__ /* * This value is bogus just to make module compilable on powerpc */ -#define PROF_ARTIFICIAL_FRAMES 3 +#define PROF_ARTIFICIAL_FRAMES 8 #endif struct profile_probe_percpu; -#ifdef __mips -/* bogus */ -#define PROF_ARTIFICIAL_FRAMES 3 -#endif - #ifdef __arm__ #define PROF_ARTIFICIAL_FRAMES 3 #endif @@ -127,8 +114,7 @@ struct profile_probe_percpu; #endif #ifdef __riscv -/* TODO: verify */ -#define PROF_ARTIFICIAL_FRAMES 10 +#define PROF_ARTIFICIAL_FRAMES 12 #endif typedef struct profile_probe { @@ -261,12 +247,15 @@ profile_probe(profile_probe_t *prof, hrtime_t late) if (frame != NULL) { if (TRAPF_USERMODE(frame)) upc = TRAPF_PC(frame); - else + else { pc = TRAPF_PC(frame); + td->t_dtrace_trapframe = frame; + } } else if (TD_IS_IDLETHREAD(td)) pc = (uintfptr_t)&cpu_idle; dtrace_probe(prof->prof_id, pc, upc, late, 0, 0); + td->t_dtrace_trapframe = NULL; } static void diff --git a/sys/cddl/dev/prototype.c b/sys/cddl/dev/prototype.c index 203d910f5e00..d9b475c67529 100644 --- a/sys/cddl/dev/prototype.c +++ b/sys/cddl/dev/prototype.c @@ -2,11 +2,8 @@ * This file is freeware. You are free to use it and add your own * license. * - * $FreeBSD$ - * */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/conf.h> diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c index d352ed177e8c..684374848290 100644 --- a/sys/cddl/dev/sdt/sdt.c +++ b/sys/cddl/dev/sdt/sdt.c @@ -20,8 +20,6 @@ * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * - * $FreeBSD$ - * */ /* @@ -39,7 +37,6 @@ * unloaded; in particular, probes may not span multiple kernel modules. */ -#include <sys/cdefs.h> #include <sys/param.h> #include <sys/systm.h> diff --git a/sys/cddl/dev/systrace/systrace.c b/sys/cddl/dev/systrace/systrace.c index 3ac3bb80596b..4969343ed06e 100644 --- a/sys/cddl/dev/systrace/systrace.c +++ b/sys/cddl/dev/systrace/systrace.c @@ -26,9 +26,6 @@ * Use is subject to license terms. */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - #include <sys/param.h> #include <sys/systm.h> #include <sys/conf.h> |