diff options
| -rw-r--r-- | sys/arm/arm/mp_machdep.c | 2 | ||||
| -rw-r--r-- | sys/arm64/arm64/mp_machdep.c | 2 | ||||
| -rw-r--r-- | sys/kern/init_main.c | 2 | ||||
| -rw-r--r-- | sys/kern/sched_4bsd.c | 54 | ||||
| -rw-r--r-- | sys/kern/sched_shim.c | 96 | ||||
| -rw-r--r-- | sys/kern/sched_ule.c | 62 | ||||
| -rw-r--r-- | sys/powerpc/aim/mp_cpudep.c | 2 | ||||
| -rw-r--r-- | sys/powerpc/booke/mp_cpudep.c | 2 | ||||
| -rw-r--r-- | sys/riscv/riscv/mp_machdep.c | 2 | ||||
| -rw-r--r-- | sys/sys/sched.h | 507 | ||||
| -rw-r--r-- | sys/x86/x86/mp_x86.c | 2 |
11 files changed, 379 insertions, 354 deletions
diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c index 9a4dc4e503e1..a745adef20bc 100644 --- a/sys/arm/arm/mp_machdep.c +++ b/sys/arm/arm/mp_machdep.c @@ -177,7 +177,7 @@ init_secondary(int cpu) pc->pc_curthread = pc->pc_idlethread; pc->pc_curpcb = pc->pc_idlethread->td_pcb; set_curthread(pc->pc_idlethread); - schedinit_ap(); + sched_init_ap(); #ifdef VFP vfp_init(); #endif diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c index 30f4b30e64a7..2883e7dc74bf 100644 --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -280,7 +280,7 @@ init_secondary(uint64_t cpu) /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pcpup->pc_curthread = pcpup->pc_idlethread; - schedinit_ap(); + sched_init_ap(); /* Initialize curpmap to match TTBR0's current setting. */ pmap0 = vmspace_pmap(&vmspace0); diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index cfca5d9d1296..829583cb687c 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -493,7 +493,7 @@ proc0_init(void *dummy __unused) * Initialise scheduler resources. * Add scheduler specific parts to proc, thread as needed. */ - schedinit(); /* scheduler gets its house in order */ + sched_init(); /* scheduler gets its house in order */ /* * Create process 0. diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 5f0246323dea..431957c048eb 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -1010,7 +1010,7 @@ sched_4bsd_sleep(struct thread *td, int pri) } static void -sched_4bsd_sswitch(struct thread *td, int flags) +sched_4bsd_switch(struct thread *td, int flags) { struct thread *newtd; struct mtx *tmtx; @@ -1842,54 +1842,4 @@ sched_4bsd_find_l2_neighbor(int cpu) return (-1); } -struct sched_instance sched_4bsd_instance = { -#define SLOT(name) .name = sched_4bsd_##name - SLOT(load), - SLOT(rr_interval), - SLOT(runnable), - SLOT(exit), - SLOT(fork), - SLOT(fork_exit), - SLOT(class), - SLOT(nice), - SLOT(ap_entry), - SLOT(exit_thread), - SLOT(estcpu), - SLOT(fork_thread), - SLOT(ithread_prio), - SLOT(lend_prio), - SLOT(lend_user_prio), - SLOT(lend_user_prio_cond), - SLOT(pctcpu), - SLOT(prio), - SLOT(sleep), - SLOT(sswitch), - SLOT(throw), - SLOT(unlend_prio), - SLOT(user_prio), - SLOT(userret_slowpath), - SLOT(add), - SLOT(choose), - SLOT(clock), - SLOT(idletd), - SLOT(preempt), - SLOT(relinquish), - SLOT(rem), - SLOT(wakeup), - SLOT(bind), - SLOT(unbind), - SLOT(is_bound), - SLOT(affinity), - SLOT(sizeof_proc), - SLOT(sizeof_thread), - SLOT(tdname), - SLOT(clear_tdname), - SLOT(find_l2_neighbor), - SLOT(init), - SLOT(init_ap), - SLOT(setup), - SLOT(initticks), - SLOT(sysinit), -#undef SLOT -}; -DECLARE_SCHEDULER(fourbsd_sched_selector, "4BSD", &sched_4bsd_instance); +DECLARE_SCHEDULER(4bsd, "4BSD"); diff --git a/sys/kern/sched_shim.c b/sys/kern/sched_shim.c index 9935c4e40aa8..8fdd12e5f84d 100644 --- a/sys/kern/sched_shim.c +++ b/sys/kern/sched_shim.c @@ -18,76 +18,10 @@ #include <sys/sched.h> #include <sys/smp.h> #include <sys/sysctl.h> -#include <machine/ifunc.h> - -const struct sched_instance *active_sched; -#define __DEFINE_SHIM(__m, __r, __n, __p, __a) \ - DEFINE_IFUNC(, __r, __n, __p) \ - { \ - return (active_sched->__m); \ - } -#define DEFINE_SHIM0(__m, __r, __n) \ - __DEFINE_SHIM(__m, __r, __n, (void), ()) -#define DEFINE_SHIM1(__m, __r, __n, __t1, __a1) \ - __DEFINE_SHIM(__m, __r, __n, (__t1 __a1), (__a1)) -#define DEFINE_SHIM2(__m, __r, __n, __t1, __a1, __t2, __a2) \ - __DEFINE_SHIM(__m, __r, __n, (__t1 __a1, __t2 __a2), (__a1, __a2)) - -DEFINE_SHIM0(load, int, sched_load) -DEFINE_SHIM0(rr_interval, int, sched_rr_interval) -DEFINE_SHIM0(runnable, bool, sched_runnable) -DEFINE_SHIM2(exit, void, sched_exit, struct proc *, p, - struct thread *, childtd) -DEFINE_SHIM2(fork, void, sched_fork, struct thread *, td, - struct thread *, childtd) -DEFINE_SHIM1(fork_exit, void, sched_fork_exit, struct thread *, td) -DEFINE_SHIM2(class, void, sched_class, struct thread *, td, int, class) -DEFINE_SHIM2(nice, void, sched_nice, struct proc *, p, int, nice) -DEFINE_SHIM0(ap_entry, void, sched_ap_entry) -DEFINE_SHIM2(exit_thread, void, sched_exit_thread, struct thread *, td, - struct thread *, child) -DEFINE_SHIM1(estcpu, u_int, sched_estcpu, struct thread *, td) -DEFINE_SHIM2(fork_thread, void, sched_fork_thread, struct thread *, td, - struct thread *, child) -DEFINE_SHIM2(ithread_prio, void, sched_ithread_prio, struct thread *, td, - u_char, prio) -DEFINE_SHIM2(lend_prio, void, sched_lend_prio, struct thread *, td, - u_char, prio) -DEFINE_SHIM2(lend_user_prio, void, sched_lend_user_prio, struct thread *, td, - u_char, pri) -DEFINE_SHIM2(lend_user_prio_cond, void, sched_lend_user_prio_cond, - struct thread *, td, u_char, pri) -DEFINE_SHIM1(pctcpu, fixpt_t, sched_pctcpu, struct thread *, td) -DEFINE_SHIM2(prio, void, sched_prio, struct thread *, td, u_char, prio) -DEFINE_SHIM2(sleep, void, sched_sleep, struct thread *, td, int, prio) -DEFINE_SHIM2(sswitch, void, sched_switch, struct thread *, td, int, flags) -DEFINE_SHIM1(throw, void, sched_throw, struct thread *, td) -DEFINE_SHIM2(unlend_prio, void, sched_unlend_prio, struct thread *, td, - u_char, prio) -DEFINE_SHIM2(user_prio, void, sched_user_prio, struct thread *, td, - u_char, prio) -DEFINE_SHIM1(userret_slowpath, void, sched_userret_slowpath, - struct thread *, td) -DEFINE_SHIM2(add, void, sched_add, struct thread *, td, int, flags) -DEFINE_SHIM0(choose, struct thread *, sched_choose) -DEFINE_SHIM2(clock, void, sched_clock, struct thread *, td, int, cnt) -DEFINE_SHIM1(idletd, void, sched_idletd, void *, dummy) -DEFINE_SHIM1(preempt, void, sched_preempt, struct thread *, td) -DEFINE_SHIM1(relinquish, void, sched_relinquish, struct thread *, td) -DEFINE_SHIM1(rem, void, sched_rem, struct thread *, td) -DEFINE_SHIM2(wakeup, void, sched_wakeup, struct thread *, td, int, srqflags) -DEFINE_SHIM2(bind, void, sched_bind, struct thread *, td, int, cpu) -DEFINE_SHIM1(unbind, void, sched_unbind, struct thread *, td) -DEFINE_SHIM1(is_bound, int, sched_is_bound, struct thread *, td) -DEFINE_SHIM1(affinity, void, sched_affinity, struct thread *, td) -DEFINE_SHIM0(sizeof_proc, int, sched_sizeof_proc) -DEFINE_SHIM0(sizeof_thread, int, sched_sizeof_thread) -DEFINE_SHIM1(tdname, char *, sched_tdname, struct thread *, td) -DEFINE_SHIM1(clear_tdname, void, sched_clear_tdname, struct thread *, td) -DEFINE_SHIM1(find_l2_neighbor, int, sched_find_l2_neighbor, int, cpu) -DEFINE_SHIM0(init_ap, void, schedinit_ap) +#include <machine/ifunc.h> +static const struct sched_instance *active_sched; SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions"); SCHED_STAT_DEFINE(ithread_preemptions, @@ -117,6 +51,18 @@ int __read_mostly dtrace_vtime_active; dtrace_vtime_switch_func_t dtrace_vtime_switch_func; #endif +/* + * Define dispatchers for all functions described by SCHED_ITF_DISPATCH(). + */ +#define DEFINE_SHIM(args, ret_type, fn, ...) \ + DEFINE_IFUNC(, ret_type, SCHED_ITF_FUNCTION_NAME(fn), \ + (SCHED_ITF_ARGS_PROTO_NAME(__VA_ARGS__))) \ + { \ + return (active_sched->SCHED_ITF_FIELD_NAME(fn)); \ + } +SCHED_ITF_DISPATCH(DEFINE_SHIM, ); +#undef DEFINE_SHIM + static char sched_name[32] = "ULE"; SET_DECLARE(sched_instance_set, struct sched_selection); @@ -155,27 +101,27 @@ sched_instance_select(void) } void -schedinit(void) +sched_init(void) { if (active_sched == NULL) panic("Cannot find scheduler %s", sched_name); - active_sched->init(); + active_sched->SCHED_ITF_FIELD_NAME(init)(); } struct cpu_group __read_mostly *cpu_top; /* CPU topology */ static void -sched_setup(void *dummy) +sched_setup(void) { cpu_top = smp_topo(); - active_sched->setup(); + active_sched->SCHED_ITF_FIELD_NAME(setup)(); } SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL); static void -sched_initticks(void *dummy) +sched_initticks(void) { - active_sched->initticks(); + active_sched->SCHED_ITF_FIELD_NAME(initticks)(); } SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, NULL); @@ -183,7 +129,7 @@ SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, static void sched_sysinit(void) { - active_sched->sysinit(); + active_sched->SCHED_ITF_FIELD_NAME(sysinit)(); } SYSINIT(sched_sysinit, SI_SUB_LAST, SI_ORDER_FIRST, sched_sysinit, NULL); diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index ea5f63311387..453e3958faf9 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -1899,14 +1899,14 @@ sched_ule_init(void) } /* - * schedinit_ap() is needed prior to calling sched_throw(NULL) to ensure that + * sched_init_ap() is needed prior to calling sched_throw(NULL) to ensure that * the pcpu requirements are met for any calls in the period between curthread * initialization and sched_throw(). One can safely add threads to the queue * before sched_throw(), for instance, as long as the thread lock is setup * correctly. * * TDQ_SELF() relies on the below sched pcpu setting; it may be used only - * after schedinit_ap(). + * after sched_init_ap(). */ static void sched_ule_init_ap(void) @@ -2320,7 +2320,7 @@ thread_unblock_switch(struct thread *td, struct mtx *mtx) * be assigned elsewhere via binding. */ static void -sched_ule_sswitch(struct thread *td, int flags) +sched_ule_switch(struct thread *td, int flags) { struct thread *newtd; struct tdq *tdq; @@ -3240,7 +3240,7 @@ sched_ule_ap_entry(void) tdq = TDQ_SELF(); - /* This should have been setup in schedinit_ap(). */ + /* This should have been setup in sched_init_ap(). */ THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq)); TDQ_LOCK(tdq); @@ -3418,58 +3418,6 @@ sched_ule_find_l2_neighbor(int cpu) } #endif -struct sched_instance sched_ule_instance = { -#define SLOT(name) .name = sched_ule_##name - SLOT(load), - SLOT(rr_interval), - SLOT(runnable), - SLOT(exit), - SLOT(fork), - SLOT(fork_exit), - SLOT(class), - SLOT(nice), - SLOT(ap_entry), - SLOT(exit_thread), - SLOT(estcpu), - SLOT(fork_thread), - SLOT(ithread_prio), - SLOT(lend_prio), - SLOT(lend_user_prio), - SLOT(lend_user_prio_cond), - SLOT(pctcpu), - SLOT(prio), - SLOT(sleep), - SLOT(sswitch), - SLOT(throw), - SLOT(unlend_prio), - SLOT(user_prio), - SLOT(userret_slowpath), - SLOT(add), - SLOT(choose), - SLOT(clock), - SLOT(idletd), - SLOT(preempt), - SLOT(relinquish), - SLOT(rem), - SLOT(wakeup), - SLOT(bind), - SLOT(unbind), - SLOT(is_bound), - SLOT(affinity), - SLOT(sizeof_proc), - SLOT(sizeof_thread), - SLOT(tdname), - SLOT(clear_tdname), - SLOT(find_l2_neighbor), - SLOT(init), - SLOT(init_ap), - SLOT(setup), - SLOT(initticks), - SLOT(sysinit), -#undef SLOT -}; -DECLARE_SCHEDULER(ule_sched_selector, "ULE", &sched_ule_instance); - static int sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) { @@ -3551,3 +3499,5 @@ SYSCTL_INT(_kern_sched_ule, OID_AUTO, always_steal, CTLFLAG_RWTUN, &always_steal, 0, "Always run the stealer from the idle thread"); #endif + +DECLARE_SCHEDULER(ule, "ULE"); diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c index 98acfc1a5c37..6bfec392f166 100644 --- a/sys/powerpc/aim/mp_cpudep.c +++ b/sys/powerpc/aim/mp_cpudep.c @@ -132,7 +132,7 @@ cpudep_ap_bootstrap(void) #endif pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb; sp = pcpup->pc_curpcb->pcb_sp; - schedinit_ap(); + sched_init_ap(); return (sp); } diff --git a/sys/powerpc/booke/mp_cpudep.c b/sys/powerpc/booke/mp_cpudep.c index a57c80102045..e38ce1bded99 100644 --- a/sys/powerpc/booke/mp_cpudep.c +++ b/sys/powerpc/booke/mp_cpudep.c @@ -83,7 +83,7 @@ cpudep_ap_bootstrap(void) #endif pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb; sp = pcpup->pc_curpcb->pcb_sp; - schedinit_ap(); + sched_init_ap(); /* XXX shouldn't the pcb_sp be checked/forced for alignment here?? */ diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c index 9c81d9ee1043..5402c6d388ed 100644 --- a/sys/riscv/riscv/mp_machdep.c +++ b/sys/riscv/riscv/mp_machdep.c @@ -180,7 +180,7 @@ init_secondary(uint64_t hart) /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pcpup->pc_curthread = pcpup->pc_idlethread; - schedinit_ap(); + sched_init_ap(); /* Setup and enable interrupts */ intr_pic_init_secondary(); diff --git a/sys/sys/sched.h b/sys/sys/sched.h index e4cf848cf641..9d8524ee6464 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -36,6 +36,11 @@ /*- * Copyright (c) 2002-2008, Jeffrey Roberson <jeff@freebsd.org> * All rights reserved. + * Copyright (c) 2026 The FreeBSD Foundation + * + * Portions of this software were developed by Olivier Certner + * <olce@FreeBSD.org> at Kumacom SARL under sponsorship from the FreeBSD + * Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -71,51 +76,316 @@ #include <sys/linker_set.h> #include <sys/sdt.h> -struct proc; -struct thread; +/* + * Scheduler interface. + * + * The scheduler interface is the list of functions that external users can + * call. It consists of two parts: Functions that must be provided directly by + * a scheduler implementation (called an "instance"), and functions with + * a common implementation (which may indirectly call instance functions). + * + * The former are declared in an abstract manner with some of the SCHED_ITF_*() + * macros (SCHED_ITF_INTERNAL() being a notable exception), described in detail + * below. A fUN() line declares a function, and the final interface function + * name is the symbol in the third argument prefixed by 'sched_'. E.g., such + * lines: + * + * FUN((__VA_ARGS__), int, load) + * FUN((__VA_ARGS__), void, switch, struct thread *, td, int, flags) + * + * lead to the C declarations (among others): + * + * int sched_load(void); + * void sched_switch(struct thread *, int); + * + * and each implementation (instance) must implement corresponding functions, + * e.g., for ULE: + * + * int sched_ule_load(void); + * void sched_ule_switch(struct thread *td, int flags); + * + * Such interface functions are grouped and commented in the SCHED_ITF*() + * macros listing them. + * + * Functions with a common implementation are declared under "Common functions" + * below as commented plain C declarations. + */ /* - * General scheduling info. + * Macros SCHED_ITF_*() define functions that must be provided by an instance. + * Please see their herald comments for details about their differences. They + * are X macros to avoid repetitions in function signatures, slot names, + * dispatch code, and scheduler instance declaration. They take as an argument + * the FUN() macro, which is expanded on each interface's function descriptor. * - * sched_load: - * Total runnable non-ithread threads in the system. + * The arguments passed to FUN() are: + * - The variable arguments to the X macro, as the first argument. + * - The function descriptor, consisting of: + * - Field name in 'struct sched_instance'. + * - Return type. + * - Function name; prefixed by 'sched_' for the publicly visible functions, + * and by 'sched_<instance_name>_' for instance's implementation. + * - Function arguments, each being represented as a pair of successive macro + * parameters, the first being the argument's type and the second its name. + * Note that FUN() is passed a variable number of arguments for flexibility. * - * sched_runnable: - * Runnable threads for this processor. + * In SCHED_ITF_*() macros, please keep the function's parameters on a separate + * line for clarity, even if everything would fit on a single line. */ -int sched_load(void); -int sched_rr_interval(void); -bool sched_runnable(void); -/* - * Proc related scheduling hooks. +/* sched_add() arguments. */ +#define SRQ_BORING 0x0000 /* No special circumstances. */ +#define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */ +#define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */ +#define SRQ_INTR 0x0004 /* It is probably urgent. */ +#define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */ +#define SRQ_BORROWING 0x0010 /* Priority updated due to prio_lend */ +#define SRQ_HOLD 0x0020 /* Return holding original td lock */ +#define SRQ_HOLDTD 0x0040 /* Return holding td lock */ + +/* + * Functions to be directly dispatched to the active scheduler instance. + * + * These are implemented using ifuncs for zero-cost dispatch. */ -void sched_exit(struct proc *p, struct thread *childtd); -void sched_fork(struct thread *td, struct thread *childtd); -void sched_fork_exit(struct thread *td); -void sched_class(struct thread *td, int class); -void sched_nice(struct proc *p, int nice); +#define SCHED_ITF_DISPATCH(FUN, ...) \ + /* \ + * General scheduling info. \ + * \ + * sched_load: \ + * Total runnable non-ithread threads in the system. \ + * \ + * sched_runnable: \ + * Runnable threads for this processor. \ + */ \ + FUN((__VA_ARGS__), int, load) \ + FUN((__VA_ARGS__), int, rr_interval) \ + FUN((__VA_ARGS__), bool, runnable) \ + \ + /* \ + * Proc related scheduling hooks. \ + */ \ + FUN((__VA_ARGS__), void, exit, \ + struct proc *, p, struct thread *, child) \ + FUN((__VA_ARGS__), void, fork, \ + struct thread *, td, struct thread *, child) \ + FUN((__VA_ARGS__), void, fork_exit, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, class, \ + struct thread *, td, int, class) \ + FUN((__VA_ARGS__), void, nice, \ + struct proc *, p, int, nice) \ + \ + /* \ + * Threads are switched in and out, block on resources, \ + * have temporary priorities inherited from their procs, \ + * and use up cpu time. \ + */ \ + FUN((__VA_ARGS__), void, ap_entry) \ + FUN((__VA_ARGS__), void, exit_thread, \ + struct thread *, td, struct thread *, child) \ + FUN((__VA_ARGS__), u_int, estcpu, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, fork_thread, \ + struct thread *, td, struct thread *, child) \ + FUN((__VA_ARGS__), void, ithread_prio, \ + struct thread *, td, u_char, prio) \ + FUN((__VA_ARGS__), void, lend_prio, \ + struct thread *, td, u_char, prio) \ + FUN((__VA_ARGS__), void, lend_user_prio, \ + struct thread *, td, u_char, pri) \ + FUN((__VA_ARGS__), void, lend_user_prio_cond, \ + struct thread *, td, u_char, pri) \ + FUN((__VA_ARGS__), fixpt_t, pctcpu, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, prio, \ + struct thread *, td, u_char, prio) \ + FUN((__VA_ARGS__), void, sleep, \ + struct thread *, td, int, prio) \ + FUN((__VA_ARGS__), void, switch, \ + struct thread *, td, int, flags) \ + FUN((__VA_ARGS__), void, throw, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, unlend_prio, \ + struct thread *, td, u_char, prio) \ + FUN((__VA_ARGS__), void, user_prio, \ + struct thread *, td, u_char, prio) \ + FUN((__VA_ARGS__), void, userret_slowpath, \ + struct thread *, td) \ + \ + /* \ + * Threads are moved on and off of run queues \ + */ \ + FUN((__VA_ARGS__), void, add, \ + struct thread *, td, int, flags) \ + FUN((__VA_ARGS__), struct thread *, choose) \ + FUN((__VA_ARGS__), void, clock, \ + struct thread *, td, int, cnt) \ + FUN((__VA_ARGS__), void, idletd, \ + void *, dummy) \ + FUN((__VA_ARGS__), void, preempt, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, relinquish, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, rem, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, wakeup, \ + struct thread *, td, int, srqflags) \ + \ + /* \ + * Binding makes cpu affinity permanent while pinning is used \ + * to temporarily hold a thread on a particular CPU. \ + */ \ + FUN((__VA_ARGS__), void, bind, \ + struct thread *, td, int, cpu) \ + FUN((__VA_ARGS__), void, unbind, \ + struct thread *, td) \ + FUN((__VA_ARGS__), int, is_bound, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, affinity, \ + struct thread *, td) \ + \ + /* \ + * These procedures tell the process data structure allocation \ + * code how many bytes to actually allocate. \ + */ \ + FUN((__VA_ARGS__), int, sizeof_proc) \ + FUN((__VA_ARGS__), int, sizeof_thread) \ + \ + /* \ + * This routine provides a consistent thread name for use with \ + * KTR graphing FUNctions. \ + */ \ + FUN((__VA_ARGS__), char *, tdname, \ + struct thread *, td) \ + FUN((__VA_ARGS__), void, clear_tdname, \ + struct thread *, td) \ + \ + /* \ + * Find an L2 neighbor of the given CPU or return -1 if none \ + * found. This does not distinguish among multiple L2 \ + * if the given CPU has more than one (it will always return \ + * the same result in that case). \ + */ \ + FUN((__VA_ARGS__), int, find_l2_neighbor, \ + int, cpu) \ + \ + /* \ + * Fixup scheduler state for secondary APs \ + */ \ + FUN((__VA_ARGS__), void, init_ap) /* - * Threads are switched in and out, block on resources, have temporary - * priorities inherited from their procs, and use up cpu time. + * Instance functions to be specifically wrapped. + * + * An explicit common implementation must be provided for these, and it is + * expected to call the instance's implementation. + */ +#define SCHED_ITF_WRAP(FUN, ...) \ + /* \ + * Initialize scheduler state for proc0 and thread0. \ + */ \ + FUN((__VA_ARGS__), void, init) + +/* + * Instance functions used internally. + * + * Functions here are not made available for external callers. + */ +#define SCHED_ITF_INTERNAL(FUN, ...) \ + /* \ + * Setup run queues ('cpu_top' filled by the shim machinery). \ + */ \ + FUN((__VA_ARGS__), void, setup) \ + \ + /* \ + * Determines time constants after stathz and hz are setup. \ + */ \ + FUN((__VA_ARGS__), void, initticks) \ + \ + /* \ + * Final steps after all the rest has been initialized. \ + */ \ + FUN((__VA_ARGS__), void, sysinit) + +/* All parts at once, but common functions. */ +#define SCHED_ITF_NOCOMMON(FUN, ...) \ + SCHED_ITF_DISPATCH(FUN, __VA_ARGS__) \ + SCHED_ITF_WRAP(FUN, __VA_ARGS__) \ + SCHED_ITF_INTERNAL(FUN, __VA_ARGS__) + +/* + * Machinery to help define FUN() macros. + */ +#define _SCHED_ITF_ARG1(a0, ...) \ + a0 +#define _SCHED_ITF_ARG9(a0, a1, a2, a3, a4, a5, a6, a7, a8, ...) \ + a8 +#define _SCHED_ITF_NARGS(...) \ + _SCHED_ITF_ARG9(__VA_ARGS__ __VA_OPT__(,) 4, ERR, 3, ERR, 2, \ + ERR, 1, ERR, 0) +#define _SCHED_ITF_NARGS_DISP(PREFIX, disp_args, full_args) \ + __CONCAT(__CONCAT(PREFIX, _), _SCHED_ITF_NARGS disp_args) \ + full_args +#define _SCHED_ITF_ARGS_PAIR_ERR \ + "Two macro arguments per interface function argument expected." +#define _SCHED_ITF_ARGS_PROTO_NAME_0() \ + void +#define _SCHED_ITF_ARGS_PROTO_NAME_1(t1, a1) \ + t1 a1 +#define _SCHED_ITF_ARGS_PROTO_NAME_2(t1, a1, t2, a2) \ + t1 a1, t2 a2 +#define _SCHED_ITF_ARGS_PROTO_NAME_3(t1, a1, t2, a2, t3, a3) \ + t1 a1, t2 a2, t3 a3 +#define _SCHED_ITF_ARGS_PROTO_NAME_4(t1, a1, t2, a2, t3, a3, t4, a4) \ + t1 a1, t2 a2, t3 a3, t4 a4 +#define _SCHED_ITF_ARGS_PROTO_NAME_ERR(...) \ + _SCHED_ITF_ARGS_PAIR_ERR +#define SCHED_ITF_ARGS_PROTO_NAME(...) \ + _SCHED_ITF_NARGS_DISP(_SCHED_ITF_ARGS_PROTO_NAME, \ + (__VA_ARGS__), (__VA_ARGS__)) +#define _SCHED_ITF_ARGS_PROTO_NONAME_0() \ + void +#define _SCHED_ITF_ARGS_PROTO_NONAME_1(t1, a1) \ + t1 +#define _SCHED_ITF_ARGS_PROTO_NONAME_2(t1, a1, t2, a2) \ + t1, t2 +#define _SCHED_ITF_ARGS_PROTO_NONAME_3(t1, a1, t2, a2, t3, a3) \ + t1, t2, t3 +#define _SCHED_ITF_ARGS_PROTO_NONAME_4(t1, a1, t2, a2, t3, a3, t4, a4) \ + t1, t2, t3, t4 +#define _SCHED_ITF_ARGS_PROTO_NONAME_ERR(...) \ + _SCHED_ITF_ARGS_PAIR_ERR +#define SCHED_ITF_ARGS_PROTO_NONAME(...) \ + _SCHED_ITF_NARGS_DISP(_SCHED_ITF_ARGS_PROTO_NONAME, \ + (__VA_ARGS__), (__VA_ARGS__)) + +#define SCHED_ITF_FIELD_NAME(fn) \ + __CONCAT(fn, _impl) +#define SCHED_ITF_FUNCTION_NAME(fn) \ + __CONCAT(sched_, fn) +#define _SCHED_ITF_IMPL_FUNCTION_NAME(sched_name, fn) \ + SCHED_ITF_FUNCTION_NAME(__CONCAT(__CONCAT(sched_name, _), \ + fn)) + +/* + * C declaration of first part of interface functions. + */ +struct proc; +struct thread; + +/* Eliding argument names to avoid collisions with C++ keywords. */ +#define _SCHED_ITF_FUN(args, ret_type, fn, ...) \ + ret_type \ + SCHED_ITF_FUNCTION_NAME(fn) \ + (SCHED_ITF_ARGS_PROTO_NONAME(__VA_ARGS__)); +SCHED_ITF_DISPATCH(_SCHED_ITF_FUN, ); +SCHED_ITF_WRAP(_SCHED_ITF_FUN, ); +#undef _SCHED_ITF_FUN + +/* + * Common functions. */ -void sched_ap_entry(void); -void sched_exit_thread(struct thread *td, struct thread *child); -u_int sched_estcpu(struct thread *td); -void sched_fork_thread(struct thread *td, struct thread *child); -void sched_ithread_prio(struct thread *td, u_char prio); -void sched_lend_prio(struct thread *td, u_char prio); -void sched_lend_user_prio(struct thread *td, u_char pri); -void sched_lend_user_prio_cond(struct thread *td, u_char pri); -fixpt_t sched_pctcpu(struct thread *td); -void sched_prio(struct thread *td, u_char prio); -void sched_sleep(struct thread *td, int prio); -void sched_switch(struct thread *td, int flags); -void sched_throw(struct thread *td); -void sched_unlend_prio(struct thread *td, u_char prio); -void sched_user_prio(struct thread *td, u_char prio); -void sched_userret_slowpath(struct thread *td); static inline void sched_userret(struct thread *td) @@ -136,43 +406,6 @@ sched_userret(struct thread *td) sched_userret_slowpath(td); } -/* - * Threads are moved on and off of run queues - */ -void sched_add(struct thread *td, int flags); -struct thread *sched_choose(void); -void sched_clock(struct thread *td, int cnt); -void sched_idletd(void *); -void sched_preempt(struct thread *td); -void sched_relinquish(struct thread *td); -void sched_rem(struct thread *td); -void sched_wakeup(struct thread *td, int srqflags); - -/* - * Binding makes cpu affinity permanent while pinning is used to temporarily - * hold a thread on a particular CPU. - */ -void sched_bind(struct thread *td, int cpu); -static __inline void sched_pin(void); -void sched_unbind(struct thread *td); -static __inline void sched_unpin(void); -int sched_is_bound(struct thread *td); -void sched_affinity(struct thread *td); - -/* - * These procedures tell the process data structure allocation code how - * many bytes to actually allocate. - */ -int sched_sizeof_proc(void); -int sched_sizeof_thread(void); - -/* - * This routine provides a consistent thread name for use with KTR graphing - * functions. - */ -char *sched_tdname(struct thread *td); -void sched_clear_tdname(struct thread *td); - static __inline void sched_pin(void) { @@ -188,19 +421,9 @@ sched_unpin(void) curthread->td_pinned--; } -void ast_scheduler(struct thread *td, int tda); - -/* sched_add arguments (formerly setrunqueue) */ -#define SRQ_BORING 0x0000 /* No special circumstances. */ -#define SRQ_YIELDING 0x0001 /* We are yielding (from mi_switch). */ -#define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */ -#define SRQ_INTR 0x0004 /* It is probably urgent. */ -#define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */ -#define SRQ_BORROWING 0x0010 /* Priority updated due to prio_lend */ -#define SRQ_HOLD 0x0020 /* Return holding original td lock */ -#define SRQ_HOLDTD 0x0040 /* Return holding td lock */ - -/* Scheduler stats. */ +/* + * Scheduler statistics. + */ #ifdef SCHED_STATS DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]); @@ -237,6 +460,7 @@ SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL); SCHED_STAT_DECLARE(ithread_demotions); SCHED_STAT_DECLARE(ithread_preemptions); +/* Dtrace. */ SDT_PROBE_DECLARE(sched, , , change__pri); SDT_PROBE_DECLARE(sched, , , dequeue); SDT_PROBE_DECLARE(sched, , , enqueue); @@ -254,95 +478,50 @@ extern dtrace_vtime_switch_func_t dtrace_vtime_switch_func; #endif /* - * Fixup scheduler state for proc0 and thread0 + * Scheduler instance structure. */ -void schedinit(void); - -/* - * Fixup scheduler state for secondary APs - */ -void schedinit_ap(void); - -/* - * Find an L2 neighbor of the given CPU or return -1 if none found. This - * does not distinguish among multiple L2 neighbors if the given CPU has - * more than one (it will always return the same result in that case). - */ -int sched_find_l2_neighbor(int cpu); - -/* - * The scheduler selection interface uses names that are reserved words in - * C++, causing problems for downstream projects that use C++ in the - * kernel. - */ -#ifndef __cplusplus - +#define _SCHED_ITF_FUN(args, ret_type, fn, ...) \ + ret_type (*SCHED_ITF_FIELD_NAME(fn)) \ + (SCHED_ITF_ARGS_PROTO_NONAME(__VA_ARGS__)); struct sched_instance { - int (*load)(void); - int (*rr_interval)(void); - bool (*runnable)(void); - void (*exit)(struct proc *p, struct thread *childtd); - void (*fork)(struct thread *td, struct thread *childtd); - void (*fork_exit)(struct thread *td); - void (*class)(struct thread *td, int class); - void (*nice)(struct proc *p, int nice); - void (*ap_entry)(void); - void (*exit_thread)(struct thread *td, struct thread *child); - u_int (*estcpu)(struct thread *td); - void (*fork_thread)(struct thread *td, struct thread *child); - void (*ithread_prio)(struct thread *td, u_char prio); - void (*lend_prio)(struct thread *td, u_char prio); - void (*lend_user_prio)(struct thread *td, u_char pri); - void (*lend_user_prio_cond)(struct thread *td, u_char pri); - fixpt_t (*pctcpu)(struct thread *td); - void (*prio)(struct thread *td, u_char prio); - void (*sleep)(struct thread *td, int prio); - void (*sswitch)(struct thread *td, int flags); - void (*throw)(struct thread *td); - void (*unlend_prio)(struct thread *td, u_char prio); - void (*user_prio)(struct thread *td, u_char prio); - void (*userret_slowpath)(struct thread *td); - void (*add)(struct thread *td, int flags); - struct thread *(*choose)(void); - void (*clock)(struct thread *td, int cnt); - void (*idletd)(void *); - void (*preempt)(struct thread *td); - void (*relinquish)(struct thread *td); - void (*rem)(struct thread *td); - void (*wakeup)(struct thread *td, int srqflags); - void (*bind)(struct thread *td, int cpu); - void (*unbind)(struct thread *td); - int (*is_bound)(struct thread *td); - void (*affinity)(struct thread *td); - int (*sizeof_proc)(void); - int (*sizeof_thread)(void); - char *(*tdname)(struct thread *td); - void (*clear_tdname)(struct thread *td); - int (*find_l2_neighbor)(int cpuid); - void (*init)(void); - void (*init_ap)(void); - void (*setup)(void); - void (*initticks)(void); - /* Scheduler init, to be called only from 'sched_shim.c'. */ - void (*sysinit)(void); + SCHED_ITF_NOCOMMON(_SCHED_ITF_FUN, ) }; +#undef _SCHED_ITF_FUN -extern const struct sched_instance *active_sched; - +/* + * Scheduler instance declaration. + */ struct sched_selection { const char *name; const struct sched_instance *instance; }; -#define DECLARE_SCHEDULER(xsel_name, xsched_name, xsched_instance) \ - static struct sched_selection xsel_name = { \ - .name = xsched_name, \ - .instance = xsched_instance, \ - }; \ - DATA_SET(sched_instance_set, xsel_name); + +#define _SCHED_DECLARE_FIELD_FUN(args, ret_type, fn, ...) \ + .SCHED_ITF_FIELD_NAME(fn) = &_SCHED_ITF_IMPL_FUNCTION_NAME( \ + _SCHED_ITF_ARG1 args, fn), + +#define DECLARE_SCHEDULER(var_name, string_name) \ + static const struct sched_instance \ + __CONCAT(_sched_instance_, var_name) = { \ + SCHED_ITF_NOCOMMON(_SCHED_DECLARE_FIELD_FUN, \ + var_name) \ + }; \ + static const struct sched_selection \ + __CONCAT(_sched_selector_, var_name) = { \ + .name = string_name, \ + .instance = &__CONCAT(_sched_instance_, var_name), \ + }; \ + DATA_SET(sched_instance_set, \ + __CONCAT(_sched_selector_, var_name)); void sched_instance_select(void); -#endif /* !__cplusplus */ +/* + * Miscellaneous. + */ + +void ast_scheduler(struct thread *td, int tda); + #endif /* _KERNEL */ @@ -359,10 +538,10 @@ struct sched_param { int sched_priority; }; +#ifndef _KERNEL /* * POSIX scheduling declarations for userland. */ -#ifndef _KERNEL #include <sys/cdefs.h> #include <sys/_timespec.h> #include <sys/_types.h> diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index c01822569030..eff18b1875af 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -1085,7 +1085,7 @@ init_secondary_tail(void) /* Initialize curthread. */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); PCPU_SET(curthread, PCPU_GET(idlethread)); - schedinit_ap(); + sched_init_ap(); mca_init(); |