config_intrhook: provide config_intrhook_drain

config_intrhook_drain will remove the hook from the list as
config_intrhook_disestablish does if the hook hasn't been called.  If it has,
config_intrhook_drain will wait for the hook to be disestablished in the normal
course (or expedited, it's up to the driver to decide how and when
to call config_intrhook_disestablish).

This is intended for removable devices that use config_intrhook and might be
attached early in boot, but that may be removed before the kernel can call the
config_intrhook or before it ends. To prevent all races, the detach routine will
need to call config_intrhook_train.

Sponsored by:		Netflix, Inc
Reviewed by:		jhb, mav, gde (in D29006 for man page)
Differential Revision:	https://reviews.freebsd.org/D29005

(cherry picked from commit e52368365db3c0a696b37bfc09d08b7093b41b57)

4 files changed, 73 insertions, 4 deletions

diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index 0b025df78061..8bf3b7c22f74 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -859,6 +859,7 @@ MLINKS+=condvar.9 cv_broadcast.9 \
 	condvar.9 cv_wait_unlock.9 \
 	condvar.9 cv_wmesg.9
 MLINKS+=config_intrhook.9 config_intrhook_disestablish.9 \
+	config_intrhook.9 config_intrhook_drain.9 \
 	config_intrhook.9 config_intrhook_establish.9 \
 	config_intrhook.9 config_intrhook_oneshot.9
 MLINKS+=contigmalloc.9 contigmalloc_domainset.9 \
diff --git a/share/man/man9/config_intrhook.9 b/share/man/man9/config_intrhook.9
index 0414ad88218a..6892a089b00f 100644
--- a/share/man/man9/config_intrhook.9
+++ b/share/man/man9/config_intrhook.9
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 10, 2017
+.Dd March 8, 2021
 .Dt CONFIG_INTRHOOK 9
 .Os
 .Sh NAME
@@ -40,6 +40,8 @@ but before root is mounted
 .Fn config_intrhook_establish "struct intr_config_hook *hook"
 .Ft void
 .Fn config_intrhook_disestablish "struct intr_config_hook *hook"
+.Ft int
+.Fn config_intrhook_drain "struct intr_config_hook *hook"
 .Ft void
 .Fn config_intrhook_oneshot "ich_func_t func" "void *arg"
 .Sh DESCRIPTION
@@ -55,6 +57,23 @@ The
 function removes the entry from the hook queue.
 .Pp
 The
+.Fn config_intrhook_drain
+function removes the entry from the hook queue in a safe way.
+If the hook is not currently active it removes
+.Fa hook
+from the hook queue and returns
+.Vt ICHS_QUEUED .
+If the hook is active, it waits for the hook to complete before returning
+.Vt ICHS_RUNNING .
+If the hook has previously completed, it returns
+.Vt ICHS_DONE .
+Because a
+.Vt config_intrhook
+is undefined prior to
+.Fn config_intrhook_establish ,
+this function may only be called after that function has returned.
+.Pp
+The
 .Fn config_intrhook_oneshot
 function schedules a function to be run as described for
 .Fn config_intrhook_establish ;
diff --git a/sys/kern/subr_autoconf.c b/sys/kern/subr_autoconf.c
index 6a998a533801..77c48485e1da 100644
--- a/sys/kern/subr_autoconf.c
+++ b/sys/kern/subr_autoconf.c
@@ -138,6 +138,7 @@ run_interrupt_driven_config_hooks()
 	while (next_to_notify != NULL) {
 		hook_entry = next_to_notify;
 		next_to_notify = TAILQ_NEXT(hook_entry, ich_links);
+		hook_entry->ich_state = ICHS_RUNNING;
 		mtx_unlock(&intr_config_hook_lock);
 		(*hook_entry->ich_func)(hook_entry->ich_arg);
 		mtx_lock(&intr_config_hook_lock);
@@ -199,6 +200,7 @@ config_intrhook_establish(struct intr_config_hook *hook)
 	TAILQ_INSERT_TAIL(&intr_config_hook_list, hook, ich_links);
 	if (next_to_notify == NULL)
 		next_to_notify = hook;
+	hook->ich_state = ICHS_QUEUED;
 	mtx_unlock(&intr_config_hook_lock);
 	if (cold == 0)
 		/*
@@ -226,12 +228,11 @@ config_intrhook_oneshot(ich_func_t func, void *arg)
 	config_intrhook_establish(&ohook->och_hook);
 }
 
-void
-config_intrhook_disestablish(struct intr_config_hook *hook)
+static void
+config_intrhook_disestablish_locked(struct intr_config_hook *hook)
 {
 	struct intr_config_hook *hook_entry;
 
-	mtx_lock(&intr_config_hook_lock);
 	TAILQ_FOREACH(hook_entry, &intr_config_hook_list, ich_links)
 		if (hook_entry == hook)
 			break;
@@ -245,8 +246,51 @@ config_intrhook_disestablish(struct intr_config_hook *hook)
 	TSRELEASE("config hooks");
 
 	/* Wakeup anyone watching the list */
+	hook->ich_state = ICHS_DONE;
 	wakeup(&intr_config_hook_list);
+}
+
+void
+config_intrhook_disestablish(struct intr_config_hook *hook)
+{
+	mtx_lock(&intr_config_hook_lock);
+	config_intrhook_disestablish_locked(hook);
+	mtx_unlock(&intr_config_hook_lock);
+}
+
+int
+config_intrhook_drain(struct intr_config_hook *hook)
+{
+	mtx_lock(&intr_config_hook_lock);
+
+	/*
+	 * The config hook has completed, so just return.
+	 */
+	if (hook->ich_state == ICHS_DONE) {
+		mtx_unlock(&intr_config_hook_lock);
+		return (ICHS_DONE);
+	}
+
+	/*
+	 * The config hook hasn't started running, just call disestablish.
+	 */
+	if (hook->ich_state == ICHS_QUEUED) {
+		config_intrhook_disestablish_locked(hook);
+		mtx_unlock(&intr_config_hook_lock);
+		return (ICHS_QUEUED);
+	}
+
+	/*
+	 * The config hook is running, so wait for it to complete and return.
+	 */
+	while (hook->ich_state != ICHS_DONE) {
+		if (msleep(&intr_config_hook_list, &intr_config_hook_lock,
+		    0, "confhd", hz) == EWOULDBLOCK) {
+			// XXX do I whine?
+		}
+	}
 	mtx_unlock(&intr_config_hook_lock);
+	return (ICHS_RUNNING);
 }
 
 #ifdef DDB
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
index 181036c98a6a..3e48a57e0f83 100644
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -467,12 +467,17 @@ typedef void (*ich_func_t)(void *_arg);
 
 struct intr_config_hook {
 	TAILQ_ENTRY(intr_config_hook) ich_links;
+	uintptr_t	ich_state;
+#define ICHS_QUEUED	0x1
+#define ICHS_RUNNING	0x2
+#define	ICHS_DONE	0x3
 	ich_func_t	ich_func;
 	void		*ich_arg;
 };
 
 int	config_intrhook_establish(struct intr_config_hook *hook);
 void	config_intrhook_disestablish(struct intr_config_hook *hook);
+int	config_intrhook_drain(struct intr_config_hook *hook);
 void	config_intrhook_oneshot(ich_func_t _func, void *_arg);
 
 #endif /* !_SYS_KERNEL_H_*/