kinst: port to riscv

Reviewed by:	markj
Approved by:	markj (mentor)
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D39884

3 files changed, 650 insertions, 0 deletions

diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.c b/sys/cddl/dev/kinst/riscv/kinst_isa.c
new file mode 100644
index 000000000000..8bd4449a027a
--- /dev/null
+++ b/sys/cddl/dev/kinst/riscv/kinst_isa.c
@@ -0,0 +1,616 @@
+/*
+ * SPDX-License-Identifier: CDDL 1.0
+ *
+ * Copyright (c) 2023 The FreeBSD Foundation
+ *
+ * This software was developed by Christos Margiolis <christos@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ */
+
+#include <sys/param.h>
+
+#include <sys/dtrace.h>
+#include <cddl/dev/dtrace/dtrace_cddl.h>
+
+#include "kinst.h"
+
+/*
+ * Per-CPU trampolines used when the interrupted thread is executing with
+ * interrupts disabled.  If an interrupt is raised while executing a trampoline,
+ * the interrupt thread cannot safely overwrite its trampoline if it hits a
+ * kinst probe while executing the interrupt handler.
+ */
+DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp);
+
+/*
+ * The double-breakpoint mechanism needs to save the current probe for the next
+ * call to kinst_invop(). As with per-CPU trampolines, this also has to be done
+ * per-CPU when interrupts are disabled.
+ */
+DPCPU_DEFINE_STATIC(struct kinst_probe *, intr_probe);
+
+#define _MATCH_REG(reg)	\
+	(offsetof(struct trapframe, tf_ ## reg) / sizeof(register_t))
+
+static int
+kinst_regoff(struct trapframe *frame, int n)
+{
+	switch (n) {
+	case 0:
+		/* There is no zero register in the trapframe structure. */
+		return (-1);
+	case 1:
+		return (_MATCH_REG(ra));
+	case 2:
+		return (_MATCH_REG(sp));
+	case 3:
+		return (_MATCH_REG(gp));
+	case 4:
+		return (_MATCH_REG(tp));
+	case 5 ... 7:
+		return (_MATCH_REG(t[n - 5]));
+	case 8 ... 9:
+		return (_MATCH_REG(s[n - 8]));
+	case 10 ... 17:
+		return (_MATCH_REG(a[n - 10]));
+	case 18 ... 27:
+		return (_MATCH_REG(s[n - 18 + 2]));
+	case 28 ... 31:
+		return (_MATCH_REG(t[n - 28 + 3]));
+	default:
+		panic("%s: unhandled register index %d", __func__, n);
+	}
+}
+
+static int
+kinst_c_regoff(struct trapframe *frame, int n)
+{
+	switch (n) {
+	case 0 ... 1:
+		return (_MATCH_REG(s[n]));
+	case 2 ... 7:
+		return (_MATCH_REG(a[n - 2]));
+	default:
+		panic("%s: unhandled register index %d", __func__, n);
+	}
+}
+
+#undef _MATCH_REG
+
+static int
+kinst_emulate(struct trapframe *frame, struct kinst_probe *kp)
+{
+	kinst_patchval_t instr = kp->kp_savedval;
+	register_t prevpc;
+	uint64_t imm;
+	uint16_t off;
+	uint8_t funct;
+
+	if (kp->kp_md.instlen == INSN_SIZE) {
+#define rs1_index	((instr & RS1_MASK) >> RS1_SHIFT)
+#define rs2_index	((instr & RS2_MASK) >> RS2_SHIFT)
+#define rd_index	((instr & RD_MASK) >> RD_SHIFT)
+#define rs1		((register_t *)frame)[kinst_regoff(frame, rs1_index)]
+#define rs2		((register_t *)frame)[kinst_regoff(frame, rs2_index)]
+#define rd		((register_t *)frame)[kinst_regoff(frame, rd_index)]
+#define rs1_lval	(rs1_index != 0 ? rs1 : 0)
+#define rs2_lval	(rs2_index != 0 ? rs2 : 0)
+		switch (instr & 0x7f) {
+		case 0b1101111: /* jal */
+			imm = 0;
+			imm |= ((instr >> 21) & 0x03ff) << 1;
+			imm |= ((instr >> 20) & 0x0001) << 11;
+			imm |= ((instr >> 12) & 0x00ff) << 12;
+			imm |= ((instr >> 31) & 0x0001) << 20;
+			if (imm & 0x0000000000100000)
+				imm |= 0xfffffffffff00000;
+			if (rd_index != 0)
+				rd = frame->tf_sepc + INSN_SIZE;
+			frame->tf_sepc += imm;
+			break;
+		case 0b1100111:	/* jalr */
+			prevpc = frame->tf_sepc;
+			imm = (instr & IMM_MASK) >> IMM_SHIFT;
+			if (imm & 0x0000000000000800)
+				imm |= 0xfffffffffffff000;
+			frame->tf_sepc = (rs1_lval + imm) & ~1;
+			if (rd_index != 0)
+				rd = prevpc + INSN_SIZE;
+			break;
+		case 0b1100011:	/* branch */
+			imm = 0;
+			imm |= ((instr >> 8) & 0x000f) << 1;
+			imm |= ((instr >> 25) & 0x003f) << 5;
+			imm |= ((instr >> 7) & 0x0001) << 11;
+			imm |= ((instr >> 31) & 0x0001) << 12;
+			if (imm & 0x0000000000001000)
+				imm |= 0xfffffffffffff000;
+			funct = (instr >> 12) & 0x07;
+			switch (funct) {
+			case 0b000:	/* beq */
+				if (rs1_lval == rs2_lval)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_SIZE;
+				break;
+			case 0b001:	/* bne */
+				if (rs1_lval != rs2_lval)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_SIZE;
+				break;
+			case 0b100:	/* blt */
+				if ((int64_t)rs1_lval < (int64_t)rs2_lval)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_SIZE;
+				break;
+			case 0b110:	/* bltu */
+				if ((uint64_t)rs1_lval < (uint64_t)rs2_lval)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_SIZE;
+				break;
+			case 0b101:	/* bge */
+				if ((int64_t)rs1_lval >= (int64_t)rs2_lval)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_SIZE;
+				break;
+			case 0b111:	/* bgeu */
+				if ((uint64_t)rs1_lval >= (uint64_t)rs2_lval)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_SIZE;
+				break;
+			}
+			break;
+		case 0b0010111:	/* auipc */
+			imm = instr & 0xfffff000;
+			rd = frame->tf_sepc +
+			    (imm & 0x0000000080000000 ?
+			    imm | 0xffffffff80000000 : imm);
+			frame->tf_sepc += INSN_SIZE;
+			break;
+		}
+#undef rs1_lval
+#undef rs2_lval
+#undef rs1
+#undef rs2
+#undef rd
+#undef rs1_index
+#undef rs2_index
+#undef rd_index
+	} else {
+		switch (instr & 0x03) {
+#define rs1	\
+	((register_t *)frame)[kinst_c_regoff(frame, (instr >> 7) & 0x07)]
+		case 0b01:
+			funct = (instr >> 13) & 0x07;
+			switch (funct) {
+			case 0b101:	/* c.j */
+				off = (instr >> 2) & 0x07ff;
+				imm = 0;
+				imm |= ((off >> 1) & 0x07) << 1;
+				imm |= ((off >> 9) & 0x01) << 4;
+				imm |= ((off >> 0) & 0x01) << 5;
+				imm |= ((off >> 5) & 0x01) << 6;
+				imm |= ((off >> 4) & 0x01) << 7;
+				imm |= ((off >> 7) & 0x03) << 8;
+				imm |= ((off >> 6) & 0x01) << 10;
+				imm |= ((off >> 10) & 0x01) << 11;
+				if (imm & 0x0000000000000800)
+					imm |= 0xfffffffffffff000;
+				frame->tf_sepc += imm;
+				break;
+			case 0b110:	/* c.beqz */
+			case 0b111:	/* c.bnez */
+				imm = 0;
+				imm |= ((instr >> 3) & 0x03) << 1;
+				imm |= ((instr >> 10) & 0x03) << 3;
+				imm |= ((instr >> 2) & 0x01) << 5;
+				imm |= ((instr >> 5) & 0x03) << 6;
+				imm |= ((instr >> 12) & 0x01) << 8;
+				if (imm & 0x0000000000000100)
+					imm |= 0xffffffffffffff00;
+				if (funct == 0b110 && rs1 == 0)
+					frame->tf_sepc += imm;
+				else if (funct == 0b111 && rs1 != 0)
+					frame->tf_sepc += imm;
+				else
+					frame->tf_sepc += INSN_C_SIZE;
+				break;
+			}
+			break;
+#undef rs1
+#define rs1_index	((instr & RD_MASK) >> RD_SHIFT)
+#define rs1		((register_t *)frame)[kinst_regoff(frame, rs1_index)]
+		case 0b10:
+			funct = (instr >> 13) & 0x07;
+			if (funct == 0b100 && rs1_index != 0) {
+				/* c.jr/c.jalr */
+				prevpc = frame->tf_sepc;
+				frame->tf_sepc = rs1;
+				if (((instr >> 12) & 0x01) != 0)
+					frame->tf_ra = prevpc + INSN_C_SIZE;
+			}
+			break;
+#undef rs1
+#undef rs1_index
+		}
+	}
+
+	return (MATCH_C_NOP);
+}
+
+static int
+kinst_jump_next_instr(struct trapframe *frame, struct kinst_probe *kp)
+{
+	frame->tf_sepc = (register_t)((uint8_t *)kp->kp_patchpoint +
+	    kp->kp_md.instlen);
+
+	return (MATCH_C_NOP);
+}
+
+static void
+kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp)
+{
+	static uint16_t nop = MATCH_C_NOP;
+	static uint32_t ebreak = MATCH_EBREAK;
+	int ilen;
+
+	ilen = kp->kp_md.instlen;
+	kinst_memcpy(tramp, &kp->kp_savedval, ilen);
+
+	/*
+	 * Since we cannot encode large displacements in a single instruction
+	 * in order to encode a far-jump back to the next instruction, and we
+	 * also cannot clobber a register inside the trampoline, we execute a
+	 * breakpoint after the copied instruction. kinst_invop() is
+	 * responsible for detecting this special case and performing the
+	 * "jump" manually.
+	 *
+	 * Add a NOP after a compressed instruction for padding.
+	 */
+	if (ilen == INSN_C_SIZE)
+		kinst_memcpy(&tramp[ilen], &nop, INSN_C_SIZE);
+
+	kinst_memcpy(&tramp[INSN_SIZE], &ebreak, INSN_SIZE);
+
+	fence_i();
+}
+
+/*
+ * There are two ways by which an instruction is traced:
+ *
+ * - By using the trampoline.
+ * - By emulating it in software (see kinst_emulate()).
+ *
+ * The trampoline is used for instructions that can be copied and executed
+ * as-is without additional modification. However, instructions that use
+ * PC-relative addressing have to be emulated, because RISC-V doesn't allow
+ * encoding of large displacements in a single instruction, and since we cannot
+ * clobber a register in order to encode the two-instruction sequence needed to
+ * create large displacements, we cannot use the trampoline at all.
+ * Fortunately, the instructions are simple enough to be emulated in just a few
+ * lines of code.
+ *
+ * The problem discussed above also means that, unlike amd64, we cannot encode
+ * a far-jump back from the trampoline to the next instruction. The mechanism
+ * employed to achieve this functionality, is to use a breakpoint instead of a
+ * jump after the copied instruction. This breakpoint is detected and handled
+ * by kinst_invop(), which performs the jump back to the next instruction
+ * manually (see kinst_jump_next_instr()).
+ */
+int
+kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch)
+{
+	solaris_cpu_t *cpu;
+	struct kinst_probe *kp;
+	uint8_t *tramp;
+
+	/*
+	 * Use per-CPU trampolines and probes if the thread executing the
+	 * instruction was executing with interrupts disabled.
+	 */
+	if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) {
+		tramp = DPCPU_GET(intr_tramp);
+		kp = DPCPU_GET(intr_probe);
+	} else {
+		tramp = curthread->t_kinst_tramp;
+		kp = curthread->t_kinst_curprobe;
+	}
+
+	/*
+	 * Detect if the breakpoint was triggered by the trampoline, and
+	 * manually set the PC to the next instruction.
+	 */
+	if (addr == (uintptr_t)(tramp + INSN_SIZE))
+		return (kinst_jump_next_instr(frame, kp));
+
+	LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) {
+		if ((uintptr_t)kp->kp_patchpoint == addr)
+			break;
+	}
+	if (kp == NULL)
+		return (0);
+
+	cpu = &solaris_cpu[curcpu];
+	cpu->cpu_dtrace_caller = addr;
+	dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0);
+	cpu->cpu_dtrace_caller = 0;
+
+	if (kp->kp_md.emulate)
+		return (kinst_emulate(frame, kp));
+
+	if (tramp == NULL) {
+		/*
+		 * A trampoline allocation failed, so this probe is
+		 * effectively disabled.  Restore the original
+		 * instruction.
+		 *
+		 * We can't safely print anything here, but the
+		 * trampoline allocator should have left a breadcrumb in
+		 * the dmesg.
+		 */
+		kinst_patch_tracepoint(kp, kp->kp_savedval);
+		frame->tf_sepc = (register_t)kp->kp_patchpoint;
+	} else {
+		kinst_trampoline_populate(kp, tramp);
+		frame->tf_sepc = (register_t)tramp;
+		if ((frame->tf_sstatus & SSTATUS_SPIE) == 0)
+			DPCPU_SET(intr_probe, kp);
+		else
+			curthread->t_kinst_curprobe = kp;
+	}
+
+	return (MATCH_C_NOP);
+}
+
+void
+kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val)
+{
+	switch (kp->kp_patchval) {
+	case KINST_C_PATCHVAL:
+		*(uint16_t *)kp->kp_patchpoint = (uint16_t)val;
+		fence_i();
+		break;
+	case KINST_PATCHVAL:
+		*kp->kp_patchpoint = val;
+		fence_i();
+		break;
+	}
+}
+
+static void
+kinst_instr_dissect(struct kinst_probe *kp, int instrsize)
+{
+	struct kinst_probe_md *kpmd;
+	kinst_patchval_t instr = kp->kp_savedval;
+	uint8_t funct;
+
+	kpmd = &kp->kp_md;
+	kpmd->instlen = instrsize;
+	kpmd->emulate = false;
+
+	/*
+	 * The following instructions use PC-relative addressing and need to be
+	 * emulated in software.
+	 */
+	if (kpmd->instlen == INSN_SIZE) {
+		switch (instr & 0x7f) {
+		case 0b1101111: /* jal */
+		case 0b1100111:	/* jalr */
+		case 0b1100011:	/* branch */
+		case 0b0010111:	/* auipc */
+			kpmd->emulate = true;
+			break;
+		}
+	} else {
+		switch (instr & 0x03) {
+		case 0b01:
+			funct = (instr >> 13) & 0x07;
+			switch (funct) {
+			case 0b101:	/* c.j */
+			case 0b110:	/* c.beqz */
+			case 0b111:	/* c.bnez */
+				kpmd->emulate = true;
+				break;
+			}
+			break;
+		case 0b10:
+			funct = (instr >> 13) & 0x07;
+			if (funct == 0b100 &&
+			    ((instr >> 7) & 0x1f) != 0 &&
+			    ((instr >> 2) & 0x1f) == 0)
+				kpmd->emulate = true;	/* c.jr/c.jalr */
+			break;
+		}
+	}
+}
+
+static bool
+kinst_instr_system(kinst_patchval_t instr)
+{
+	if (dtrace_match_opcode(instr, MATCH_C_EBREAK, MASK_C_EBREAK) ||
+	    (instr & 0x7f) == 0b1110011)
+		return (true);
+
+	return (false);
+}
+
+static bool
+kinst_instr_lr(kinst_patchval_t instr)
+{
+	if (dtrace_match_opcode(instr, MATCH_LR_W, MASK_LR_W) ||
+	    dtrace_match_opcode(instr, MATCH_LR_D, MASK_LR_D))
+		return (true);
+
+	return (false);
+}
+
+static bool
+kinst_instr_sc(kinst_patchval_t instr)
+{
+	if (dtrace_match_opcode(instr, MATCH_SC_W, MASK_SC_W) ||
+	    dtrace_match_opcode(instr, MATCH_SC_D, MASK_SC_D))
+		return (true);
+
+	return (false);
+}
+
+int
+kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
+    void *opaque)
+{
+	struct kinst_probe *kp;
+	dtrace_kinst_probedesc_t *pd;
+	const char *func;
+	kinst_patchval_t *insn, v;
+	uint8_t *instr, *limit;
+	int instrsize, n, off;
+	bool lrsc_block, store_found, ret_found;
+
+	pd = opaque;
+	func = symval->name;
+
+	if (kinst_excluded(func))
+		return (0);
+	if (strcmp(func, pd->kpd_func) != 0)
+		return (0);
+
+	instr = (uint8_t *)(symval->value);
+	limit = (uint8_t *)(symval->value + symval->size);
+	if (instr >= limit)
+		return (0);
+
+	/* Check for the usual function prologue. */
+	for (insn = (kinst_patchval_t *)instr;
+	    insn < (kinst_patchval_t *)limit; insn++) {
+		if (dtrace_instr_sdsp(&insn) || dtrace_instr_c_sdsp(&insn))
+			store_found = true;
+		else if (dtrace_instr_ret(&insn) || dtrace_instr_c_ret(&insn))
+			ret_found = true;
+		if (store_found && ret_found)
+			break;
+	}
+	if (!store_found || !ret_found)
+		return (0);
+
+	n = 0;
+	lrsc_block = false;
+	while (instr < limit) {
+		instrsize = dtrace_instr_size(instr);
+		off = (int)(instr - (uint8_t *)symval->value);
+
+		/*
+		 * Avoid undefined behavior (i.e simply casting `*instr` to
+		 * `kinst_patchval_t`) in case the pointer is unaligned.
+		 * memcpy() can safely operate on unaligned pointers.
+		 */
+		memcpy(&v, instr, sizeof(kinst_patchval_t));
+
+		/* Skip SYSTEM instructions. */
+		if (kinst_instr_system(v))
+			goto cont;
+
+		/*
+		 * Skip LR/SC blocks used to build atomic operations. If a
+		 * breakpoint is placed in a LR/SC block, the loop becomes
+		 * unconstrained. In this case we violate the operation and the
+		 * loop might fail on some implementations (see section 8.3 of
+		 * the RISC-V unprivileged spec).
+		 */
+		if (kinst_instr_lr(v))
+			lrsc_block = true;
+		else if (kinst_instr_sc(v)) {
+			lrsc_block = false;
+			goto cont;
+		}
+		if (lrsc_block)
+			goto cont;
+
+		if (pd->kpd_off != -1 && off != pd->kpd_off)
+			goto cont;
+
+		/*
+		 * Prevent separate dtrace(1) instances from creating copies of
+		 * the same probe.
+		 */
+		LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) {
+			if (strcmp(kp->kp_func, func) == 0 &&
+			    strtol(kp->kp_name, NULL, 10) == off)
+				return (0);
+		}
+		if (++n > KINST_PROBETAB_MAX) {
+			KINST_LOG("probe list full: %d entries", n);
+			return (ENOMEM);
+		}
+		kp = malloc(sizeof(struct kinst_probe), M_KINST,
+		    M_WAITOK | M_ZERO);
+		kp->kp_func = func;
+		snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off);
+		kp->kp_patchpoint = (kinst_patchval_t *)instr;
+		kp->kp_savedval = v;
+		if (instrsize == INSN_SIZE)
+			kp->kp_patchval = KINST_PATCHVAL;
+		else
+			kp->kp_patchval = KINST_C_PATCHVAL;
+
+		kinst_instr_dissect(kp, instrsize);
+		kinst_probe_create(kp, lf);
+cont:
+		instr += instrsize;
+	}
+	if (lrsc_block)
+		KINST_LOG("warning: unterminated LR/SC block");
+
+	return (0);
+}
+
+int
+kinst_md_init(void)
+{
+	uint8_t *tramp;
+	int cpu;
+
+	CPU_FOREACH(cpu) {
+		tramp = kinst_trampoline_alloc(M_WAITOK);
+		if (tramp == NULL)
+			return (ENOMEM);
+		DPCPU_ID_SET(cpu, intr_tramp, tramp);
+	}
+
+	return (0);
+}
+
+void
+kinst_md_deinit(void)
+{
+	uint8_t *tramp;
+	int cpu;
+
+	CPU_FOREACH(cpu) {
+		tramp = DPCPU_ID_GET(cpu, intr_tramp);
+		if (tramp != NULL) {
+			kinst_trampoline_dealloc(tramp);
+			DPCPU_ID_SET(cpu, intr_tramp, NULL);
+		}
+	}
+}
+
+/*
+ * Exclude machine-dependent functions that are not safe-to-trace.
+ */
+bool
+kinst_md_excluded(const char *name)
+{
+	if (strcmp(name, "cpu_exception_handler") == 0 ||
+            strcmp(name, "cpu_exception_handler_supervisor") == 0 ||
+            strcmp(name, "cpu_exception_handler_user") == 0 ||
+            strcmp(name, "do_trap_supervisor") == 0 ||
+            strcmp(name, "do_trap_user") == 0)
+                return (true);
+
+	return (false);
+}
diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.h b/sys/cddl/dev/kinst/riscv/kinst_isa.h
new file mode 100644
index 000000000000..bd7c2b3b8dcf
--- /dev/null
+++ b/sys/cddl/dev/kinst/riscv/kinst_isa.h
@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: CDDL 1.0
+ *
+ * Copyright (c) 2023 The FreeBSD Foundation
+ *
+ * This software was developed by Christos Margiolis <christos@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ */
+
+#ifndef _KINST_ISA_H_
+#define _KINST_ISA_H_
+
+#include <machine/riscvreg.h>
+#include <machine/encoding.h>
+
+#define KINST_PATCHVAL		MATCH_EBREAK
+#define KINST_C_PATCHVAL	MATCH_C_EBREAK
+
+/*
+ * The trampoline contains [instruction, [nop padding], ebreak].
+ */
+#define KINST_TRAMP_SIZE	8
+
+typedef uint32_t kinst_patchval_t;
+
+struct kinst_probe_md {
+	int	instlen;	/* original instr len */
+	bool	emulate;	/* emulate in sw */
+};
+
+#endif /* _KINST_ISA_H_ */
diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile
index 31beadb4d890..e38e2687e8a7 100644
--- a/sys/modules/dtrace/Makefile
+++ b/sys/modules/dtrace/Makefile
@@ -21,6 +21,9 @@ SUBDIR+=	fasttrap systrace_linux
 SUBDIR+=	systrace_linux32
 SUBDIR+=	kinst
 .endif
+.if ${MACHINE_CPUARCH} == "riscv"
+SUBDIR+=	kinst
+.endif
 .if ${MACHINE_CPUARCH} == "powerpc"
 SUBDIR+=	fasttrap
 .endif