sched: New scheduler interface definition scheme

Define the scheduler interface once and for all (in 'sys/sys/sched.h')
and remove all code duplication related to it (function signatures, slot
names, dispatch, scheduler instance declaration), making it easier to
modify the interface or to add new schedulers.

This is implemented by defining the interface as X macros, which are
passed a macro (and additional arguments for it) that is "called" with
a variable number of arguments describing a single function of the
interface.  The convention used for a function's parameters is that each
parameter is represented with two macro arguments, the first one being
the type and the second one being the name.  Helper macros allow to
process arguments described in this convention in order to generate
a list of arguments for function definitions (currently, up to
4 function parameters).  The chosen convention eliminates the need for
any specific declaration of functions depending on their number of
arguments.

DECLARE_SCHEDULER() has been changed to automatically declare and fill
a 'struct sched_instance' structure.  Its number of parameters has been
reduced to the minimum, and the symbol part used to produce distinct
structure names can now start with a digit (useful for 4BSD).

The new implementation has the following additional benefits:
1. The name of implementations for the interface functions is imposed
   (same name as the interface function but with the scheduler symbol
   that was passed to DECLARE_SCHEDULER() added just after the 'sched_'
   prefix, e.g., for ULE, 'sched_ule_*()') and checked at compile-time.
2. Missing functions in an implementation are detected at compile-time
   (and not at runtime with a NULL dereference).
3. Failures to fill correctly the 'struct sched_instance' object
   associated to an implementation are eliminated.

Preserve comments/categories of interface functions by moving comments
that were in the old explicit interface declarations to the SCHED_ITF*()
macros, and while here, marginally improve some (and move the common
documentation for sched_initticks() from the implementations to the
interface).  Several of them are not exact, but this will be fixed in
a separate commit.

Make the active scheduler instance object internal, as it is not used
outside of the shim implementation and one of the purposes of the
interface is also to hide the actual implementation.  If the need
arises, it would be easy to add scheduler-specific routines as new
fields in 'struct sched_instance' and to extend the machinery to
automatically assign routines returning EOPNOTSUPP to these fields, and
then to provide a macro to register non-trivial implementation, instead
of making the current instance object visible again.

Additionally, in order to avoid collision with C and C++ keywords:
1. Fields in 'struct sched_instance' are now suffixed with '_impl', so
   that, e.g., 'switch()' does not require special treatment anymore and
   the corresponding implementations can be renamed back to
   `sched_*_switch()`.  This suffix avoids any collision with existing
   C++ keywords as new functions are added to the interface.  (See
   SCHED_ITF_FIELD_NAME().)
2. Actual interface functions' C declarations do not list parameter
   names anymore (sched_class() was causing a problem in C++ with its
   'class' parameter).
allowing to remove the bracketing with '#ifndef __cplusplus'.

Reviewed by:    mchoo, kib
Fixes:          ce38acee8d0b ("Add kern/sched_shim.c") (+ some followups)
MFC after:      1 week
Sponsored by:   The FreeBSD Foundation
Differential Revision:  https://reviews.freebsd.org/D59988

11 files changed, 379 insertions, 354 deletions

diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c
index 9a4dc4e503e1..a745adef20bc 100644
--- a/sys/arm/arm/mp_machdep.c
+++ b/sys/arm/arm/mp_machdep.c
@@ -177,7 +177,7 @@ init_secondary(int cpu)
 	pc->pc_curthread = pc->pc_idlethread;
 	pc->pc_curpcb = pc->pc_idlethread->td_pcb;
 	set_curthread(pc->pc_idlethread);
-	schedinit_ap();
+	sched_init_ap();
 #ifdef VFP
 	vfp_init();
 #endif
diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c
index 30f4b30e64a7..2883e7dc74bf 100644
--- a/sys/arm64/arm64/mp_machdep.c
+++ b/sys/arm64/arm64/mp_machdep.c
@@ -280,7 +280,7 @@ init_secondary(uint64_t cpu)
 	/* Initialize curthread */
 	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
 	pcpup->pc_curthread = pcpup->pc_idlethread;
-	schedinit_ap();
+	sched_init_ap();
 
 	/* Initialize curpmap to match TTBR0's current setting. */
 	pmap0 = vmspace_pmap(&vmspace0);
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index cfca5d9d1296..829583cb687c 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -493,7 +493,7 @@ proc0_init(void *dummy __unused)
 	 * Initialise scheduler resources.
 	 * Add scheduler specific parts to proc, thread as needed.
 	 */
-	schedinit();	/* scheduler gets its house in order */
+	sched_init();	/* scheduler gets its house in order */
 
 	/*
 	 * Create process 0.
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 5f0246323dea..431957c048eb 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -1010,7 +1010,7 @@ sched_4bsd_sleep(struct thread *td, int pri)
 }
 
 static void
-sched_4bsd_sswitch(struct thread *td, int flags)
+sched_4bsd_switch(struct thread *td, int flags)
 {
 	struct thread *newtd;
 	struct mtx *tmtx;
@@ -1842,54 +1842,4 @@ sched_4bsd_find_l2_neighbor(int cpu)
 	return (-1);
 }
 
-struct sched_instance sched_4bsd_instance = {
-#define	SLOT(name) .name = sched_4bsd_##name
-	SLOT(load),
-	SLOT(rr_interval),
-	SLOT(runnable),
-	SLOT(exit),
-	SLOT(fork),
-	SLOT(fork_exit),
-	SLOT(class),
-	SLOT(nice),
-	SLOT(ap_entry),
-	SLOT(exit_thread),
-	SLOT(estcpu),
-	SLOT(fork_thread),
-	SLOT(ithread_prio),
-	SLOT(lend_prio),
-	SLOT(lend_user_prio),
-	SLOT(lend_user_prio_cond),
-	SLOT(pctcpu),
-	SLOT(prio),
-	SLOT(sleep),
-	SLOT(sswitch),
-	SLOT(throw),
-	SLOT(unlend_prio),
-	SLOT(user_prio),
-	SLOT(userret_slowpath),
-	SLOT(add),
-	SLOT(choose),
-	SLOT(clock),
-	SLOT(idletd),
-	SLOT(preempt),
-	SLOT(relinquish),
-	SLOT(rem),
-	SLOT(wakeup),
-	SLOT(bind),
-	SLOT(unbind),
-	SLOT(is_bound),
-	SLOT(affinity),
-	SLOT(sizeof_proc),
-	SLOT(sizeof_thread),
-	SLOT(tdname),
-	SLOT(clear_tdname),
-	SLOT(find_l2_neighbor),
-	SLOT(init),
-	SLOT(init_ap),
-	SLOT(setup),
-	SLOT(initticks),
-	SLOT(sysinit),
-#undef SLOT
-};
-DECLARE_SCHEDULER(fourbsd_sched_selector, "4BSD", &sched_4bsd_instance);
+DECLARE_SCHEDULER(4bsd, "4BSD");
diff --git a/sys/kern/sched_shim.c b/sys/kern/sched_shim.c
index 9935c4e40aa8..8fdd12e5f84d 100644
--- a/sys/kern/sched_shim.c
+++ b/sys/kern/sched_shim.c
@@ -18,76 +18,10 @@
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/sysctl.h>
-#include <machine/ifunc.h>
-
-const struct sched_instance *active_sched;
 
-#define	__DEFINE_SHIM(__m, __r, __n, __p, __a)	\
-	DEFINE_IFUNC(, __r, __n, __p)		\
-	{					\
-		return (active_sched->__m);	\
-	}
-#define	DEFINE_SHIM0(__m, __r, __n)	\
-    __DEFINE_SHIM(__m, __r, __n, (void), ())
-#define	DEFINE_SHIM1(__m, __r, __n, __t1, __a1)	\
-    __DEFINE_SHIM(__m, __r, __n, (__t1 __a1), (__a1))
-#define	DEFINE_SHIM2(__m, __r, __n, __t1, __a1, __t2, __a2)	\
-    __DEFINE_SHIM(__m, __r, __n, (__t1 __a1, __t2 __a2), (__a1, __a2))
-
-DEFINE_SHIM0(load, int, sched_load)
-DEFINE_SHIM0(rr_interval, int, sched_rr_interval)
-DEFINE_SHIM0(runnable, bool, sched_runnable)
-DEFINE_SHIM2(exit, void, sched_exit, struct proc *, p,
-    struct thread *, childtd)
-DEFINE_SHIM2(fork, void, sched_fork, struct thread *, td,
-    struct thread *, childtd)
-DEFINE_SHIM1(fork_exit, void, sched_fork_exit, struct thread *, td)
-DEFINE_SHIM2(class, void, sched_class, struct thread *, td, int, class)
-DEFINE_SHIM2(nice, void, sched_nice, struct proc *, p, int, nice)
-DEFINE_SHIM0(ap_entry, void, sched_ap_entry)
-DEFINE_SHIM2(exit_thread, void, sched_exit_thread, struct thread *, td,
-    struct thread *, child)
-DEFINE_SHIM1(estcpu, u_int, sched_estcpu, struct thread *, td)
-DEFINE_SHIM2(fork_thread, void, sched_fork_thread, struct thread *, td,
-    struct thread *, child)
-DEFINE_SHIM2(ithread_prio, void, sched_ithread_prio, struct thread *, td,
-    u_char, prio)
-DEFINE_SHIM2(lend_prio, void, sched_lend_prio, struct thread *, td,
-    u_char, prio)
-DEFINE_SHIM2(lend_user_prio, void, sched_lend_user_prio, struct thread *, td,
-    u_char, pri)
-DEFINE_SHIM2(lend_user_prio_cond, void, sched_lend_user_prio_cond,
-    struct thread *, td, u_char, pri)
-DEFINE_SHIM1(pctcpu, fixpt_t, sched_pctcpu, struct thread *, td)
-DEFINE_SHIM2(prio, void, sched_prio, struct thread *, td, u_char, prio)
-DEFINE_SHIM2(sleep, void, sched_sleep, struct thread *, td, int, prio)
-DEFINE_SHIM2(sswitch, void, sched_switch, struct thread *, td, int, flags)
-DEFINE_SHIM1(throw, void, sched_throw, struct thread *, td)
-DEFINE_SHIM2(unlend_prio, void, sched_unlend_prio, struct thread *, td,
-    u_char, prio)
-DEFINE_SHIM2(user_prio, void, sched_user_prio, struct thread *, td,
-    u_char, prio)
-DEFINE_SHIM1(userret_slowpath, void, sched_userret_slowpath,
-    struct thread *, td)
-DEFINE_SHIM2(add, void, sched_add, struct thread *, td, int, flags)
-DEFINE_SHIM0(choose, struct thread *, sched_choose)
-DEFINE_SHIM2(clock, void, sched_clock, struct thread *, td, int, cnt)
-DEFINE_SHIM1(idletd, void, sched_idletd, void *, dummy)
-DEFINE_SHIM1(preempt, void, sched_preempt, struct thread *, td)
-DEFINE_SHIM1(relinquish, void, sched_relinquish, struct thread *, td)
-DEFINE_SHIM1(rem, void, sched_rem, struct thread *, td)
-DEFINE_SHIM2(wakeup, void, sched_wakeup, struct thread *, td, int, srqflags)
-DEFINE_SHIM2(bind, void, sched_bind, struct thread *, td, int, cpu)
-DEFINE_SHIM1(unbind, void, sched_unbind, struct thread *, td)
-DEFINE_SHIM1(is_bound, int, sched_is_bound, struct thread *, td)
-DEFINE_SHIM1(affinity, void, sched_affinity, struct thread *, td)
-DEFINE_SHIM0(sizeof_proc, int, sched_sizeof_proc)
-DEFINE_SHIM0(sizeof_thread, int, sched_sizeof_thread)
-DEFINE_SHIM1(tdname, char *, sched_tdname, struct thread *, td)
-DEFINE_SHIM1(clear_tdname, void, sched_clear_tdname, struct thread *, td)
-DEFINE_SHIM1(find_l2_neighbor, int, sched_find_l2_neighbor, int, cpu)
-DEFINE_SHIM0(init_ap, void, schedinit_ap)
+#include <machine/ifunc.h>
 
+static const struct sched_instance *active_sched;
 
 SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions");
 SCHED_STAT_DEFINE(ithread_preemptions,
@@ -117,6 +51,18 @@ int __read_mostly		dtrace_vtime_active;
 dtrace_vtime_switch_func_t	dtrace_vtime_switch_func;
 #endif
 
+/*
+ * Define dispatchers for all functions described by SCHED_ITF_DISPATCH().
+ */
+#define DEFINE_SHIM(args, ret_type, fn, ...)				\
+	DEFINE_IFUNC(, ret_type, SCHED_ITF_FUNCTION_NAME(fn),		\
+	    (SCHED_ITF_ARGS_PROTO_NAME(__VA_ARGS__)))			\
+	{								\
+		return (active_sched->SCHED_ITF_FIELD_NAME(fn));	\
+	}
+SCHED_ITF_DISPATCH(DEFINE_SHIM, );
+#undef DEFINE_SHIM
+
 static char sched_name[32] = "ULE";
 
 SET_DECLARE(sched_instance_set, struct sched_selection);
@@ -155,27 +101,27 @@ sched_instance_select(void)
 }
 
 void
-schedinit(void)
+sched_init(void)
 {
 	if (active_sched == NULL)
 		panic("Cannot find scheduler %s", sched_name);
-	active_sched->init();
+	active_sched->SCHED_ITF_FIELD_NAME(init)();
 }
 
 struct cpu_group __read_mostly *cpu_top;		/* CPU topology */
 
 static void
-sched_setup(void *dummy)
+sched_setup(void)
 {
 	cpu_top = smp_topo();
-	active_sched->setup();
+	active_sched->SCHED_ITF_FIELD_NAME(setup)();
 }
 SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL);
 
 static void
-sched_initticks(void *dummy)
+sched_initticks(void)
 {
-	active_sched->initticks();
+	active_sched->SCHED_ITF_FIELD_NAME(initticks)();
 }
 SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
     NULL);
@@ -183,7 +129,7 @@ SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
 static void
 sched_sysinit(void)
 {
-	active_sched->sysinit();
+	active_sched->SCHED_ITF_FIELD_NAME(sysinit)();
 }
 SYSINIT(sched_sysinit, SI_SUB_LAST, SI_ORDER_FIRST, sched_sysinit, NULL);
 
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index ea5f63311387..453e3958faf9 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1899,14 +1899,14 @@ sched_ule_init(void)
 }
 
 /*
- * schedinit_ap() is needed prior to calling sched_throw(NULL) to ensure that
+ * sched_init_ap() is needed prior to calling sched_throw(NULL) to ensure that
  * the pcpu requirements are met for any calls in the period between curthread
  * initialization and sched_throw().  One can safely add threads to the queue
  * before sched_throw(), for instance, as long as the thread lock is setup
  * correctly.
  *
  * TDQ_SELF() relies on the below sched pcpu setting; it may be used only
- * after schedinit_ap().
+ * after sched_init_ap().
  */
 static void
 sched_ule_init_ap(void)
@@ -2320,7 +2320,7 @@ thread_unblock_switch(struct thread *td, struct mtx *mtx)
  * be assigned elsewhere via binding.
  */
 static void
-sched_ule_sswitch(struct thread *td, int flags)
+sched_ule_switch(struct thread *td, int flags)
 {
 	struct thread *newtd;
 	struct tdq *tdq;
@@ -3240,7 +3240,7 @@ sched_ule_ap_entry(void)
 
 	tdq = TDQ_SELF();
 
-	/* This should have been setup in schedinit_ap(). */
+	/* This should have been setup in sched_init_ap(). */
 	THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq));
 
 	TDQ_LOCK(tdq);
@@ -3418,58 +3418,6 @@ sched_ule_find_l2_neighbor(int cpu)
 }
 #endif
 
-struct sched_instance sched_ule_instance = {
-#define	SLOT(name) .name = sched_ule_##name
-	SLOT(load),
-	SLOT(rr_interval),
-	SLOT(runnable),
-	SLOT(exit),
-	SLOT(fork),
-	SLOT(fork_exit),
-	SLOT(class),
-	SLOT(nice),
-	SLOT(ap_entry),
-	SLOT(exit_thread),
-	SLOT(estcpu),
-	SLOT(fork_thread),
-	SLOT(ithread_prio),
-	SLOT(lend_prio),
-	SLOT(lend_user_prio),
-	SLOT(lend_user_prio_cond),
-	SLOT(pctcpu),
-	SLOT(prio),
-	SLOT(sleep),
-	SLOT(sswitch),
-	SLOT(throw),
-	SLOT(unlend_prio),
-	SLOT(user_prio),
-	SLOT(userret_slowpath),
-	SLOT(add),
-	SLOT(choose),
-	SLOT(clock),
-	SLOT(idletd),
-	SLOT(preempt),
-	SLOT(relinquish),
-	SLOT(rem),
-	SLOT(wakeup),
-	SLOT(bind),
-	SLOT(unbind),
-	SLOT(is_bound),
-	SLOT(affinity),
-	SLOT(sizeof_proc),
-	SLOT(sizeof_thread),
-	SLOT(tdname),
-	SLOT(clear_tdname),
-	SLOT(find_l2_neighbor),
-	SLOT(init),
-	SLOT(init_ap),
-	SLOT(setup),
-	SLOT(initticks),
-	SLOT(sysinit),
-#undef SLOT
-};
-DECLARE_SCHEDULER(ule_sched_selector, "ULE", &sched_ule_instance);
-
 static int
 sysctl_kern_quantum(SYSCTL_HANDLER_ARGS)
 {
@@ -3551,3 +3499,5 @@ SYSCTL_INT(_kern_sched_ule, OID_AUTO, always_steal, CTLFLAG_RWTUN,
     &always_steal, 0,
     "Always run the stealer from the idle thread");
 #endif
+
+DECLARE_SCHEDULER(ule, "ULE");
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index 98acfc1a5c37..6bfec392f166 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -132,7 +132,7 @@ cpudep_ap_bootstrap(void)
 #endif
 	pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
 	sp = pcpup->pc_curpcb->pcb_sp;
-	schedinit_ap();
+	sched_init_ap();
 
 	return (sp);
 }
diff --git a/sys/powerpc/booke/mp_cpudep.c b/sys/powerpc/booke/mp_cpudep.c
index a57c80102045..e38ce1bded99 100644
--- a/sys/powerpc/booke/mp_cpudep.c
+++ b/sys/powerpc/booke/mp_cpudep.c
@@ -83,7 +83,7 @@ cpudep_ap_bootstrap(void)
 #endif
 	pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
 	sp = pcpup->pc_curpcb->pcb_sp;
-	schedinit_ap();
+	sched_init_ap();
 
 	/* XXX shouldn't the pcb_sp be checked/forced for alignment here?? */
 
diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c
index 9c81d9ee1043..5402c6d388ed 100644
--- a/sys/riscv/riscv/mp_machdep.c
+++ b/sys/riscv/riscv/mp_machdep.c
@@ -180,7 +180,7 @@ init_secondary(uint64_t hart)
 	/* Initialize curthread */
 	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
 	pcpup->pc_curthread = pcpup->pc_idlethread;
-	schedinit_ap();
+	sched_init_ap();
 
 	/* Setup and enable interrupts */
 	intr_pic_init_secondary();
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index e4cf848cf641..9d8524ee6464 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -36,6 +36,11 @@
 /*-
  * Copyright (c) 2002-2008, Jeffrey Roberson <jeff@freebsd.org>
  * All rights reserved.
+ * Copyright (c) 2026 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Olivier Certner
+ * <olce@FreeBSD.org> at Kumacom SARL under sponsorship from the FreeBSD
+ * Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -71,51 +76,316 @@
 #include <sys/linker_set.h>
 #include <sys/sdt.h>
 
-struct proc;
-struct thread;
+/*
+ * Scheduler interface.
+ *
+ * The scheduler interface is the list of functions that external users can
+ * call.  It consists of two parts: Functions that must be provided directly by
+ * a scheduler implementation (called an "instance"), and functions with
+ * a common implementation (which may indirectly call instance functions).
+ *
+ * The former are declared in an abstract manner with some of the SCHED_ITF_*()
+ * macros (SCHED_ITF_INTERNAL() being a notable exception), described in detail
+ * below.  A fUN() line declares a function, and the final interface function
+ * name is the symbol in the third argument prefixed by 'sched_'.  E.g., such
+ * lines:
+ *
+ * FUN((__VA_ARGS__), int, load)
+ * FUN((__VA_ARGS__), void, switch, struct thread *, td, int, flags)
+ *
+ * lead to the C declarations (among others):
+ *
+ * int sched_load(void);
+ * void sched_switch(struct thread *, int);
+ *
+ * and each implementation (instance) must implement corresponding functions,
+ * e.g., for ULE:
+ *
+ * int sched_ule_load(void);
+ * void sched_ule_switch(struct thread *td, int flags);
+ *
+ * Such interface functions are grouped and commented in the SCHED_ITF*()
+ * macros listing them.
+ *
+ * Functions with a common implementation are declared under "Common functions"
+ * below as commented plain C declarations.
+ */
 
 /*
- * General scheduling info.
+ * Macros SCHED_ITF_*() define functions that must be provided by an instance.
+ * Please see their herald comments for details about their differences.  They
+ * are X macros to avoid repetitions in function signatures, slot names,
+ * dispatch code, and scheduler instance declaration.  They take as an argument
+ * the FUN() macro, which is expanded on each interface's function descriptor.
  *
- * sched_load:
- *	Total runnable non-ithread threads in the system.
+ * The arguments passed to FUN() are:
+ * - The variable arguments to the X macro, as the first argument.
+ * - The function descriptor, consisting of:
+ *   - Field name in 'struct sched_instance'.
+ *   - Return type.
+ *   - Function name; prefixed by 'sched_' for the publicly visible functions,
+ *     and by 'sched_<instance_name>_' for instance's implementation.
+ *   - Function arguments, each being represented as a pair of successive macro
+ *     parameters, the first being the argument's type and the second its name.
+ * Note that FUN() is passed a variable number of arguments for flexibility.
  *
- * sched_runnable:
- *	Runnable threads for this processor.
+ * In SCHED_ITF_*() macros, please keep the function's parameters on a separate
+ * line for clarity, even if everything would fit on a single line.
  */
-int	sched_load(void);
-int	sched_rr_interval(void);
-bool	sched_runnable(void);
 
-/* 
- * Proc related scheduling hooks.
+/* sched_add() arguments. */
+#define	SRQ_BORING	0x0000		/* No special circumstances. */
+#define	SRQ_YIELDING	0x0001		/* We are yielding (from mi_switch). */
+#define	SRQ_OURSELF	0x0002		/* It is ourself (from mi_switch). */
+#define	SRQ_INTR	0x0004		/* It is probably urgent. */
+#define	SRQ_PREEMPTED	0x0008		/* has been preempted.. be kind */
+#define	SRQ_BORROWING	0x0010		/* Priority updated due to prio_lend */
+#define	SRQ_HOLD	0x0020		/* Return holding original td lock */
+#define	SRQ_HOLDTD	0x0040		/* Return holding td lock */
+
+/*
+ * Functions to be directly dispatched to the active scheduler instance.
+ *
+ * These are implemented using ifuncs for zero-cost dispatch.
  */
-void	sched_exit(struct proc *p, struct thread *childtd);
-void	sched_fork(struct thread *td, struct thread *childtd);
-void	sched_fork_exit(struct thread *td);
-void	sched_class(struct thread *td, int class);
-void	sched_nice(struct proc *p, int nice);
+#define SCHED_ITF_DISPATCH(FUN, ...)					\
+	/*								\
+	 * General scheduling info.					\
+	 *								\
+	 * sched_load:							\
+	 *	Total runnable non-ithread threads in the system.	\
+	 *								\
+	 * sched_runnable:						\
+	 *	Runnable threads for this processor.			\
+	 */								\
+	FUN((__VA_ARGS__), int, load)					\
+	FUN((__VA_ARGS__), int, rr_interval)				\
+	FUN((__VA_ARGS__), bool, runnable)				\
+									\
+	/*								\
+	 * Proc related scheduling hooks.				\
+	 */								\
+	FUN((__VA_ARGS__), void, exit,					\
+	    struct proc *, p, struct thread *, child)			\
+	FUN((__VA_ARGS__), void, fork,					\
+	    struct thread *, td, struct thread *, child)		\
+	FUN((__VA_ARGS__), void, fork_exit,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, class,					\
+	    struct thread *, td, int, class)				\
+	FUN((__VA_ARGS__), void, nice,					\
+	    struct proc *, p, int, nice)				\
+									\
+	/*								\
+	 * Threads are switched in and out, block on resources,		\
+	 * have temporary priorities inherited from their procs,	\
+	 * and use up cpu time.						\
+	 */								\
+	FUN((__VA_ARGS__), void, ap_entry)				\
+	FUN((__VA_ARGS__), void, exit_thread,				\
+	    struct thread *, td, struct thread *, child)		\
+	FUN((__VA_ARGS__), u_int, estcpu,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, fork_thread,				\
+	    struct thread *, td, struct thread *, child)		\
+	FUN((__VA_ARGS__), void, ithread_prio,				\
+	    struct thread *, td, u_char, prio)				\
+	FUN((__VA_ARGS__), void, lend_prio,				\
+	    struct thread *, td, u_char, prio)				\
+	FUN((__VA_ARGS__), void, lend_user_prio,			\
+	    struct thread *, td, u_char, pri)				\
+	FUN((__VA_ARGS__), void, lend_user_prio_cond,			\
+	    struct thread *, td, u_char, pri)				\
+	FUN((__VA_ARGS__), fixpt_t, pctcpu,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, prio,					\
+	    struct thread *, td, u_char, prio)				\
+	FUN((__VA_ARGS__), void, sleep,					\
+	    struct thread *, td, int, prio)				\
+	FUN((__VA_ARGS__), void, switch,				\
+	    struct thread *, td, int, flags)				\
+	FUN((__VA_ARGS__), void, throw,					\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, unlend_prio,				\
+	    struct thread *, td, u_char, prio)				\
+	FUN((__VA_ARGS__), void, user_prio,				\
+	    struct thread *, td, u_char, prio)				\
+	FUN((__VA_ARGS__), void, userret_slowpath,			\
+	    struct thread *, td)					\
+									\
+	/*								\
+	 * Threads are moved on and off of run queues			\
+	 */								\
+	FUN((__VA_ARGS__), void, add,					\
+	    struct thread *, td, int, flags)				\
+	FUN((__VA_ARGS__), struct thread *, choose)			\
+	FUN((__VA_ARGS__), void, clock,					\
+	    struct thread *, td, int, cnt)				\
+	FUN((__VA_ARGS__), void, idletd,				\
+	    void *, dummy)						\
+	FUN((__VA_ARGS__), void, preempt,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, relinquish,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, rem,					\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, wakeup,				\
+	    struct thread *, td, int, srqflags)				\
+									\
+	/*								\
+	 * Binding makes cpu affinity permanent while pinning is used	\
+	 * to temporarily hold a thread on a particular CPU.		\
+	 */								\
+	FUN((__VA_ARGS__), void, bind,					\
+	    struct thread *, td, int, cpu)				\
+	FUN((__VA_ARGS__), void, unbind,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), int, is_bound,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, affinity,				\
+	    struct thread *, td)					\
+									\
+	/*								\
+	 * These procedures tell the process data structure allocation  \
+	 * code how many bytes to actually allocate.			\
+	 */								\
+	FUN((__VA_ARGS__), int, sizeof_proc)				\
+	FUN((__VA_ARGS__), int, sizeof_thread)				\
+									\
+	/*								\
+	 * This routine provides a consistent thread name for use with	\
+	 * KTR graphing FUNctions.					\
+	 */								\
+	FUN((__VA_ARGS__), char *, tdname,				\
+	    struct thread *, td)					\
+	FUN((__VA_ARGS__), void, clear_tdname,				\
+	    struct thread *, td)					\
+									\
+	/*								\
+	 * Find an L2 neighbor of the given CPU or return -1 if none	\
+	 * found.  This does not distinguish among multiple L2		\
+	 * if the given CPU has more than one (it will always return	\
+	 * the same result in that case).				\
+	 */								\
+	FUN((__VA_ARGS__), int,	find_l2_neighbor,			\
+	    int, cpu)							\
+									\
+	/*								\
+	 * Fixup scheduler state for secondary APs			\
+	 */								\
+	FUN((__VA_ARGS__), void, init_ap)
 
 /*
- * Threads are switched in and out, block on resources, have temporary
- * priorities inherited from their procs, and use up cpu time.
+ * Instance functions to be specifically wrapped.
+ *
+ * An explicit common implementation must be provided for these, and it is
+ * expected to call the instance's implementation.
+ */
+#define SCHED_ITF_WRAP(FUN, ...)					\
+	/*								\
+	 * Initialize scheduler state for proc0 and thread0.		\
+	 */								\
+	FUN((__VA_ARGS__), void, init)
+
+/*
+ * Instance functions used internally.
+ *
+ * Functions here are not made available for external callers.
+ */
+#define SCHED_ITF_INTERNAL(FUN, ...)					\
+	/*								\
+	 * Setup run queues ('cpu_top' filled by the shim machinery).   \
+	 */								\
+	FUN((__VA_ARGS__), void, setup)					\
+									\
+	/*								\
+	 * Determines time constants after stathz and hz are setup.	\
+	 */								\
+	FUN((__VA_ARGS__), void, initticks)				\
+									\
+	/*								\
+	 * Final steps after all the rest has been initialized.		\
+	 */								\
+	FUN((__VA_ARGS__), void, sysinit)
+
+/* All parts at once, but common functions. */
+#define SCHED_ITF_NOCOMMON(FUN, ...)					\
+	SCHED_ITF_DISPATCH(FUN, __VA_ARGS__)				\
+	SCHED_ITF_WRAP(FUN, __VA_ARGS__)				\
+	SCHED_ITF_INTERNAL(FUN, __VA_ARGS__)
+
+/*
+ * Machinery to help define FUN() macros.
+ */
+#define _SCHED_ITF_ARG1(a0, ...)					\
+	a0
+#define _SCHED_ITF_ARG9(a0, a1, a2, a3, a4, a5, a6, a7, a8, ...)	\
+	a8
+#define _SCHED_ITF_NARGS(...)						\
+	_SCHED_ITF_ARG9(__VA_ARGS__ __VA_OPT__(,) 4, ERR, 3, ERR, 2,	\
+	    ERR, 1, ERR, 0)
+#define _SCHED_ITF_NARGS_DISP(PREFIX, disp_args, full_args)		\
+	__CONCAT(__CONCAT(PREFIX, _), _SCHED_ITF_NARGS disp_args)	\
+		full_args
+#define _SCHED_ITF_ARGS_PAIR_ERR					\
+	"Two macro arguments per interface function argument expected."
+#define _SCHED_ITF_ARGS_PROTO_NAME_0()					\
+	void
+#define _SCHED_ITF_ARGS_PROTO_NAME_1(t1, a1)				\
+	t1 a1
+#define _SCHED_ITF_ARGS_PROTO_NAME_2(t1, a1, t2, a2)			\
+	t1 a1, t2 a2
+#define _SCHED_ITF_ARGS_PROTO_NAME_3(t1, a1, t2, a2, t3, a3)		\
+	t1 a1, t2 a2, t3 a3
+#define _SCHED_ITF_ARGS_PROTO_NAME_4(t1, a1, t2, a2, t3, a3, t4, a4)	\
+	t1 a1, t2 a2, t3 a3, t4 a4
+#define _SCHED_ITF_ARGS_PROTO_NAME_ERR(...)				\
+	_SCHED_ITF_ARGS_PAIR_ERR
+#define SCHED_ITF_ARGS_PROTO_NAME(...)					\
+	_SCHED_ITF_NARGS_DISP(_SCHED_ITF_ARGS_PROTO_NAME,		\
+	    (__VA_ARGS__), (__VA_ARGS__))
+#define _SCHED_ITF_ARGS_PROTO_NONAME_0()				\
+	void
+#define _SCHED_ITF_ARGS_PROTO_NONAME_1(t1, a1)				\
+	t1
+#define _SCHED_ITF_ARGS_PROTO_NONAME_2(t1, a1, t2, a2)			\
+	t1, t2
+#define _SCHED_ITF_ARGS_PROTO_NONAME_3(t1, a1, t2, a2, t3, a3)		\
+	t1, t2, t3
+#define _SCHED_ITF_ARGS_PROTO_NONAME_4(t1, a1, t2, a2, t3, a3, t4, a4)	\
+	t1, t2, t3, t4
+#define _SCHED_ITF_ARGS_PROTO_NONAME_ERR(...)				\
+	_SCHED_ITF_ARGS_PAIR_ERR
+#define SCHED_ITF_ARGS_PROTO_NONAME(...)				\
+	_SCHED_ITF_NARGS_DISP(_SCHED_ITF_ARGS_PROTO_NONAME,		\
+	    (__VA_ARGS__), (__VA_ARGS__))
+
+#define SCHED_ITF_FIELD_NAME(fn)					\
+	__CONCAT(fn, _impl)
+#define SCHED_ITF_FUNCTION_NAME(fn)					\
+	__CONCAT(sched_, fn)
+#define _SCHED_ITF_IMPL_FUNCTION_NAME(sched_name, fn)			\
+	SCHED_ITF_FUNCTION_NAME(__CONCAT(__CONCAT(sched_name, _),	\
+	    fn))
+
+/*
+ * C declaration of first part of interface functions.
+ */
+struct proc;
+struct thread;
+
+/* Eliding argument names to avoid collisions with C++ keywords. */
+#define _SCHED_ITF_FUN(args, ret_type, fn, ...)				\
+	ret_type							\
+	SCHED_ITF_FUNCTION_NAME(fn)					\
+		(SCHED_ITF_ARGS_PROTO_NONAME(__VA_ARGS__));
+SCHED_ITF_DISPATCH(_SCHED_ITF_FUN, );
+SCHED_ITF_WRAP(_SCHED_ITF_FUN, );
+#undef _SCHED_ITF_FUN
+
+/*
+ * Common functions.
  */
-void	sched_ap_entry(void);
-void	sched_exit_thread(struct thread *td, struct thread *child);
-u_int	sched_estcpu(struct thread *td);
-void	sched_fork_thread(struct thread *td, struct thread *child);
-void	sched_ithread_prio(struct thread *td, u_char prio);
-void	sched_lend_prio(struct thread *td, u_char prio);
-void	sched_lend_user_prio(struct thread *td, u_char pri);
-void	sched_lend_user_prio_cond(struct thread *td, u_char pri);
-fixpt_t	sched_pctcpu(struct thread *td);
-void	sched_prio(struct thread *td, u_char prio);
-void	sched_sleep(struct thread *td, int prio);
-void	sched_switch(struct thread *td, int flags);
-void	sched_throw(struct thread *td);
-void	sched_unlend_prio(struct thread *td, u_char prio);
-void	sched_user_prio(struct thread *td, u_char prio);
-void	sched_userret_slowpath(struct thread *td);
 
 static inline void
 sched_userret(struct thread *td)
@@ -136,43 +406,6 @@ sched_userret(struct thread *td)
 		sched_userret_slowpath(td);
 }
 
-/*
- * Threads are moved on and off of run queues
- */
-void	sched_add(struct thread *td, int flags);
-struct thread *sched_choose(void);
-void	sched_clock(struct thread *td, int cnt);
-void	sched_idletd(void *);
-void	sched_preempt(struct thread *td);
-void	sched_relinquish(struct thread *td);
-void	sched_rem(struct thread *td);
-void	sched_wakeup(struct thread *td, int srqflags);
-
-/*
- * Binding makes cpu affinity permanent while pinning is used to temporarily
- * hold a thread on a particular CPU.
- */
-void	sched_bind(struct thread *td, int cpu);
-static __inline void sched_pin(void);
-void	sched_unbind(struct thread *td);
-static __inline void sched_unpin(void);
-int	sched_is_bound(struct thread *td);
-void	sched_affinity(struct thread *td);
-
-/*
- * These procedures tell the process data structure allocation code how
- * many bytes to actually allocate.
- */
-int	sched_sizeof_proc(void);
-int	sched_sizeof_thread(void);
-
-/*
- * This routine provides a consistent thread name for use with KTR graphing
- * functions.
- */
-char	*sched_tdname(struct thread *td);
-void	sched_clear_tdname(struct thread *td);
-
 static __inline void
 sched_pin(void)
 {
@@ -188,19 +421,9 @@ sched_unpin(void)
 	curthread->td_pinned--;
 }
 
-void ast_scheduler(struct thread *td, int tda);
-
-/* sched_add arguments (formerly setrunqueue) */
-#define	SRQ_BORING	0x0000		/* No special circumstances. */
-#define	SRQ_YIELDING	0x0001		/* We are yielding (from mi_switch). */
-#define	SRQ_OURSELF	0x0002		/* It is ourself (from mi_switch). */
-#define	SRQ_INTR	0x0004		/* It is probably urgent. */
-#define	SRQ_PREEMPTED	0x0008		/* has been preempted.. be kind */
-#define	SRQ_BORROWING	0x0010		/* Priority updated due to prio_lend */
-#define	SRQ_HOLD	0x0020		/* Return holding original td lock */
-#define	SRQ_HOLDTD	0x0040		/* Return holding td lock */
-
-/* Scheduler stats. */
+/*
+ * Scheduler statistics.
+ */
 #ifdef SCHED_STATS
 DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]);
 
@@ -237,6 +460,7 @@ SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL);
 SCHED_STAT_DECLARE(ithread_demotions);
 SCHED_STAT_DECLARE(ithread_preemptions);
 
+/* Dtrace. */
 SDT_PROBE_DECLARE(sched, , , change__pri);
 SDT_PROBE_DECLARE(sched, , , dequeue);
 SDT_PROBE_DECLARE(sched, , , enqueue);
@@ -254,95 +478,50 @@ extern dtrace_vtime_switch_func_t dtrace_vtime_switch_func;
 #endif
 
 /*
- * Fixup scheduler state for proc0 and thread0
+ * Scheduler instance structure.
  */
-void schedinit(void);
-
-/*
- * Fixup scheduler state for secondary APs
- */
-void schedinit_ap(void);
-
-/*
- * Find an L2 neighbor of the given CPU or return -1 if none found.  This
- * does not distinguish among multiple L2 neighbors if the given CPU has
- * more than one (it will always return the same result in that case).
- */
-int sched_find_l2_neighbor(int cpu);
-
-/*
- * The scheduler selection interface uses names that are reserved words in
- * C++, causing problems for downstream projects that use C++ in the
- * kernel.
- */
-#ifndef __cplusplus
-
+#define _SCHED_ITF_FUN(args, ret_type, fn, ...)				\
+	ret_type (*SCHED_ITF_FIELD_NAME(fn))				\
+	(SCHED_ITF_ARGS_PROTO_NONAME(__VA_ARGS__));
 struct sched_instance {
-	int	(*load)(void);
-	int	(*rr_interval)(void);
-	bool	(*runnable)(void);
-	void	(*exit)(struct proc *p, struct thread *childtd);
-	void	(*fork)(struct thread *td, struct thread *childtd);
-	void	(*fork_exit)(struct thread *td);
-	void	(*class)(struct thread *td, int class);
-	void	(*nice)(struct proc *p, int nice);
-	void	(*ap_entry)(void);
-	void	(*exit_thread)(struct thread *td, struct thread *child);
-	u_int	(*estcpu)(struct thread *td);
-	void	(*fork_thread)(struct thread *td, struct thread *child);
-	void	(*ithread_prio)(struct thread *td, u_char prio);
-	void	(*lend_prio)(struct thread *td, u_char prio);
-	void	(*lend_user_prio)(struct thread *td, u_char pri);
-	void	(*lend_user_prio_cond)(struct thread *td, u_char pri);
-	fixpt_t	(*pctcpu)(struct thread *td);
-	void	(*prio)(struct thread *td, u_char prio);
-	void	(*sleep)(struct thread *td, int prio);
-	void	(*sswitch)(struct thread *td, int flags);
-	void	(*throw)(struct thread *td);
-	void	(*unlend_prio)(struct thread *td, u_char prio);
-	void	(*user_prio)(struct thread *td, u_char prio);
-	void	(*userret_slowpath)(struct thread *td);
-	void	(*add)(struct thread *td, int flags);
-	struct thread *(*choose)(void);
-	void	(*clock)(struct thread *td, int cnt);
-	void	(*idletd)(void *);
-	void	(*preempt)(struct thread *td);
-	void	(*relinquish)(struct thread *td);
-	void	(*rem)(struct thread *td);
-	void	(*wakeup)(struct thread *td, int srqflags);
-	void	(*bind)(struct thread *td, int cpu);
-	void	(*unbind)(struct thread *td);
-	int	(*is_bound)(struct thread *td);
-	void	(*affinity)(struct thread *td);
-	int	(*sizeof_proc)(void);
-	int	(*sizeof_thread)(void);
-	char	*(*tdname)(struct thread *td);
-	void	(*clear_tdname)(struct thread *td);
-	int	(*find_l2_neighbor)(int cpuid);
-	void	(*init)(void);
-	void	(*init_ap)(void);
-	void	(*setup)(void);
-	void	(*initticks)(void);
-	/* Scheduler init, to be called only from 'sched_shim.c'. */
-	void	(*sysinit)(void);
+	SCHED_ITF_NOCOMMON(_SCHED_ITF_FUN, )
 };
+#undef _SCHED_ITF_FUN
 
-extern const struct sched_instance *active_sched;
-
+/*
+ * Scheduler instance declaration.
+ */
 struct sched_selection {
 	const char *name;
 	const struct sched_instance *instance;
 };
-#define	DECLARE_SCHEDULER(xsel_name, xsched_name, xsched_instance)		\
-	static struct sched_selection xsel_name = {				\
-		.name = xsched_name,						\
-		.instance = xsched_instance,				\
-	};									\
-	DATA_SET(sched_instance_set, xsel_name);
+
+#define _SCHED_DECLARE_FIELD_FUN(args, ret_type, fn, ...)		\
+	.SCHED_ITF_FIELD_NAME(fn) = &_SCHED_ITF_IMPL_FUNCTION_NAME(	\
+	    _SCHED_ITF_ARG1 args, fn),
+
+#define	DECLARE_SCHEDULER(var_name, string_name)			\
+	static const struct sched_instance				\
+	__CONCAT(_sched_instance_, var_name) = {			\
+		SCHED_ITF_NOCOMMON(_SCHED_DECLARE_FIELD_FUN,		\
+		    var_name)						\
+	};								\
+	static const struct sched_selection				\
+	__CONCAT(_sched_selector_, var_name) = {			\
+		.name = string_name,					\
+		.instance = &__CONCAT(_sched_instance_, var_name),	\
+	};								\
+	DATA_SET(sched_instance_set,					\
+	    __CONCAT(_sched_selector_, var_name));
 
 void sched_instance_select(void);
 
-#endif /* !__cplusplus */
+/*
+ * Miscellaneous.
+ */
+
+void ast_scheduler(struct thread *td, int tda);
+
 
 #endif /* _KERNEL */
 
@@ -359,10 +538,10 @@ struct sched_param {
         int     sched_priority;
 };
 
+#ifndef _KERNEL
 /*
  * POSIX scheduling declarations for userland.
  */
-#ifndef _KERNEL
 #include <sys/cdefs.h>
 #include <sys/_timespec.h>
 #include <sys/_types.h>
diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c
index c01822569030..eff18b1875af 100644
--- a/sys/x86/x86/mp_x86.c
+++ b/sys/x86/x86/mp_x86.c
@@ -1085,7 +1085,7 @@ init_secondary_tail(void)
 	/* Initialize curthread. */
 	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
 	PCPU_SET(curthread, PCPU_GET(idlethread));
-	schedinit_ap();
+	sched_init_ap();
 
 	mca_init();