MFp4: Large set of CAM inprovements.

- Unify bus reset/probe sequence. Whenever bus attached at boot or later,
CAM will automatically reset and scan it. It allows to remove duplicate
code from many drivers.
- Any bus, attached before CAM completed it's boot-time initialization,
will equally join to the process, delaying boot if needed.
- New kern.cam.boot_delay loader tunable should help controllers that
are still unable to register their buses in time (such as slow USB/
PCCard/ CardBus devices), by adding one more event to wait on boot.
- To allow synchronization between different CAM levels, concept of
requests priorities was extended. Priorities now split between several
"run levels". Device can be freezed at specified level, allowing higher
priority requests to pass. For example, no payload requests allowed,
until PMP driver enable port. ATA XPT negotiate transfer parameters,
periph driver configure caching and so on.
- Frozen requests are no more counted by request allocation scheduler.
It fixes deadlocks, when frozen low priority payload requests occupying
slots, required by higher levels to manage theit execution.
- Two last changes were holding proper ATA reinitialization and error
recovery implementation. Now it is done: SATA controllers and Port
Multipliers now implement automatic hot-plug and should correctly
recover from timeouts and bus resets.
- Improve SCSI error recovery for devices on buses without automatic sense
reporting, such as ATAPI or USB. For example, it allows CAM to wait, while
CD drive loads disk, instead of immediately return error status.
- Decapitalize diagnostic messages and make them more readable and sensible.
- Teach PMP driver to limit maximum speed on fan-out ports.
- Make boot wait for PMP scan completes, and make rescan more reliable.
- Fix pass driver, to return CCB to user level in case of error.
- Increase number of retries in cd driver, as device may return several UAs.

1 files changed, 284 insertions, 376 deletions

diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index 88ee30916755..d966a9fb5bd4 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -102,6 +102,8 @@ struct xpt_softc {
 
 	/* queue for handling async rescan requests. */
 	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
+	int buses_to_config;
+	int buses_config_done;
 
 	/* Registered busses */
 	TAILQ_HEAD(,cam_eb)	xpt_busses;
@@ -109,6 +111,9 @@ struct xpt_softc {
 
 	struct intr_config_hook	*xpt_config_hook;
 
+	int			boot_delay;
+	struct callout 		boot_callout;
+
 	struct mtx		xpt_topo_lock;
 	struct mtx		xpt_lock;
 };
@@ -145,6 +150,10 @@ typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 /* Transport layer configuration information */
 static struct xpt_softc xsoftc;
 
+TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
+SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
+           &xsoftc.boot_delay, 0, "Bus registration wait time");
+
 /* Queues for our software interrupt handler */
 typedef TAILQ_HEAD(cam_isrq, ccb_hdr) cam_isrq_t;
 typedef TAILQ_HEAD(cam_simq, cam_sim) cam_simq_t;
@@ -210,11 +219,12 @@ static path_id_t xptnextfreepathid(void);
 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 static union ccb *xpt_get_ccb(struct cam_ed *device);
 static void	 xpt_run_dev_allocq(struct cam_eb *bus);
+static void	 xpt_run_dev_sendq(struct cam_eb *bus);
 static timeout_t xpt_release_devq_timeout;
 static void	 xpt_release_simq_timeout(void *arg) __unused;
 static void	 xpt_release_bus(struct cam_eb *bus);
-static void	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
-					 int run_queue);
+static void	 xpt_release_devq_device(struct cam_ed *dev, cam_rl rl,
+		    u_int count, int run_queue);
 static struct cam_et*
 		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 static void	 xpt_release_target(struct cam_et *target);
@@ -224,11 +234,8 @@ static struct cam_et*
 		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 static struct cam_ed*
 		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
-static xpt_busfunc_t	xptconfigbuscountfunc;
-static xpt_busfunc_t	xptconfigfunc;
 static void	 xpt_config(void *arg);
 static xpt_devicefunc_t xptpassannouncefunc;
-static void	 xpt_finishconfig(struct cam_periph *periph, union ccb *ccb);
 static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
 static void	 xptpoll(struct cam_sim *sim);
 static void	 camisr(void *);
@@ -270,6 +277,7 @@ static xpt_busfunc_t	xptdefbusfunc;
 static xpt_targetfunc_t	xptdeftargetfunc;
 static xpt_devicefunc_t	xptdefdevicefunc;
 static xpt_periphfunc_t	xptdefperiphfunc;
+static void		xpt_finishconfig_task(void *context, int pending);
 static int		xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg);
 static int		xpt_for_all_devices(xpt_devicefunc_t *tr_func,
 					    void *arg);
@@ -285,19 +293,19 @@ static xpt_devicefunc_t	xptsetasyncfunc;
 static xpt_busfunc_t	xptsetasyncbusfunc;
 static cam_status	xptregister(struct cam_periph *periph,
 				    void *arg);
-static __inline int xpt_schedule_dev_allocq(struct cam_eb *bus,
-					    struct cam_ed *dev);
 static __inline int periph_is_queued(struct cam_periph *periph);
 static __inline int device_is_alloc_queued(struct cam_ed *device);
 static __inline int device_is_send_queued(struct cam_ed *device);
-static __inline int dev_allocq_is_runnable(struct cam_devq *devq);
 
 static __inline int
 xpt_schedule_dev_allocq(struct cam_eb *bus, struct cam_ed *dev)
 {
 	int retval;
 
-	if (dev->ccbq.devq_openings > 0) {
+	if ((dev->drvq.entries > 0) &&
+	    (dev->ccbq.devq_openings > 0) &&
+	    (cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
+		CAMQ_GET_PRIO(&dev->drvq))) == 0)) {
 		/*
 		 * The priority of a device waiting for CCB resources
 		 * is that of the the highest priority peripheral driver
@@ -305,7 +313,7 @@ xpt_schedule_dev_allocq(struct cam_eb *bus, struct cam_ed *dev)
 		 */
 		retval = xpt_schedule_dev(&bus->sim->devq->alloc_queue,
 					  &dev->alloc_ccb_entry.pinfo,
-					  CAMQ_GET_HEAD(&dev->drvq)->priority);
+					  CAMQ_GET_PRIO(&dev->drvq));
 	} else {
 		retval = 0;
 	}
@@ -318,7 +326,9 @@ xpt_schedule_dev_sendq(struct cam_eb *bus, struct cam_ed *dev)
 {
 	int	retval;
 
-	if (dev->ccbq.dev_openings > 0) {
+	if ((dev->ccbq.queue.entries > 0) &&
+	    (dev->ccbq.dev_openings > 0) &&
+	    (cam_ccbq_frozen_top(&dev->ccbq) == 0)) {
 		/*
 		 * The priority of a device waiting for controller
 		 * resources is that of the the highest priority CCB
@@ -327,7 +337,7 @@ xpt_schedule_dev_sendq(struct cam_eb *bus, struct cam_ed *dev)
 		retval =
 		    xpt_schedule_dev(&bus->sim->devq->send_queue,
 				     &dev->send_ccb_entry.pinfo,
-				     CAMQ_GET_HEAD(&dev->ccbq.queue)->priority);
+				     CAMQ_GET_PRIO(&dev->ccbq.queue));
 	} else {
 		retval = 0;
 	}
@@ -352,19 +362,6 @@ device_is_send_queued(struct cam_ed *device)
 	return (device->send_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX);
 }
 
-static __inline int
-dev_allocq_is_runnable(struct cam_devq *devq)
-{
-	/*
-	 * Have work to do.
-	 * Have space to do more work.
-	 * Allowed to do work.
-	 */
-	return ((devq->alloc_queue.qfrozen_cnt == 0)
-	     && (devq->alloc_queue.entries > 0)
-	     && (devq->alloc_openings > 0));
-}
-
 static void
 xpt_periph_init()
 {
@@ -818,45 +815,42 @@ cam_module_event_handler(module_t mod, int what, void *arg)
 	return 0;
 }
 
+static void
+xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
+{
+
+	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
+		xpt_free_path(done_ccb->ccb_h.path);
+		xpt_free_ccb(done_ccb);
+	} else {
+		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
+		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
+	}
+	xpt_release_boot();
+}
+
 /* thread to handle bus rescans */
 static void
 xpt_scanner_thread(void *dummy)
 {
-	cam_isrq_t	queue;
 	union ccb	*ccb;
 	struct cam_sim	*sim;
 
+	xpt_lock_buses();
 	for (;;) {
-		/*
-		 * Wait for a rescan request to come in.  When it does, splice
-		 * it onto a queue from local storage so that the xpt lock
-		 * doesn't need to be held while the requests are being
-		 * processed.
-		 */
-		xpt_lock_buses();
 		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
 			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
 			       "ccb_scanq", 0);
-		TAILQ_INIT(&queue);
-		TAILQ_CONCAT(&queue, &xsoftc.ccb_scanq, sim_links.tqe);
-		xpt_unlock_buses();
-
-		while ((ccb = (union ccb *)TAILQ_FIRST(&queue)) != NULL) {
-			TAILQ_REMOVE(&queue, &ccb->ccb_h, sim_links.tqe);
+		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
+			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
+			xpt_unlock_buses();
 
 			sim = ccb->ccb_h.path->bus->sim;
 			CAM_SIM_LOCK(sim);
-
-			if( ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD )
-				ccb->ccb_h.func_code = XPT_SCAN_BUS;
-			else
-				ccb->ccb_h.func_code = XPT_SCAN_LUN;
-			ccb->ccb_h.cbfcnp = xptdone;
-			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_NORMAL);
-			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
-			xpt_free_path(ccb->ccb_h.path);
-			xpt_free_ccb(ccb);
+			xpt_action(ccb);
 			CAM_SIM_UNLOCK(sim);
+
+			xpt_lock_buses();
 		}
 	}
 }
@@ -866,21 +860,30 @@ xpt_rescan(union ccb *ccb)
 {
 	struct ccb_hdr *hdr;
 
-	/*
-	 * Don't make duplicate entries for the same paths.
-	 */
+	/* Prepare request */
+	if(ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD)
+		ccb->ccb_h.func_code = XPT_SCAN_BUS;
+	else
+		ccb->ccb_h.func_code = XPT_SCAN_LUN;
+	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
+	ccb->ccb_h.cbfcnp = xpt_rescan_done;
+	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
+	/* Don't make duplicate entries for the same paths. */
 	xpt_lock_buses();
-	TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
-		if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
-			wakeup(&xsoftc.ccb_scanq);
-			xpt_unlock_buses();
-			xpt_print(ccb->ccb_h.path, "rescan already queued\n");
-			xpt_free_path(ccb->ccb_h.path);
-			xpt_free_ccb(ccb);
-			return;
+	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
+		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
+			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
+				wakeup(&xsoftc.ccb_scanq);
+				xpt_unlock_buses();
+				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
+				xpt_free_path(ccb->ccb_h.path);
+				xpt_free_ccb(ccb);
+				return;
+			}
 		}
 	}
 	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
+	xsoftc.buses_to_config++;
 	wakeup(&xsoftc.ccb_scanq);
 	xpt_unlock_buses();
 }
@@ -923,10 +926,9 @@ xpt_init(void *dummy)
 	if (xpt_sim == NULL)
 		return (ENOMEM);
 
-	xpt_sim->max_ccbs = 16;
-
 	mtx_lock(&xsoftc.xpt_lock);
 	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
+		mtx_unlock(&xsoftc.xpt_lock);
 		printf("xpt_init: xpt_bus_register failed with status %#x,"
 		       " failing attach\n", status);
 		return (EINVAL);
@@ -940,6 +942,7 @@ xpt_init(void *dummy)
 	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 				      CAM_TARGET_WILDCARD,
 				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
+		mtx_unlock(&xsoftc.xpt_lock);
 		printf("xpt_init: xpt_create_path failed with status %#x,"
 		       " failing attach\n", status);
 		return (EINVAL);
@@ -949,7 +952,8 @@ xpt_init(void *dummy)
 			 path, NULL, 0, xpt_sim);
 	xpt_free_path(path);
 	mtx_unlock(&xsoftc.xpt_lock);
-
+	/* Install our software interrupt handlers */
+	swi_add(NULL, "cambio", camisr, NULL, SWI_CAMBIO, INTR_MPSAFE, &cambio_ih);
 	/*
 	 * Register a callback for when interrupts are enabled.
 	 */
@@ -961,7 +965,6 @@ xpt_init(void *dummy)
 		       "- failing attach\n");
 		return (ENOMEM);
 	}
-
 	xsoftc.xpt_config_hook->ich_func = xpt_config;
 	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
 		free (xsoftc.xpt_config_hook, M_CAMXPT);
@@ -969,13 +972,6 @@ xpt_init(void *dummy)
 		       "- failing attach\n");
 	}
 
-	/* fire up rescan thread */
-	if (kproc_create(xpt_scanner_thread, NULL, NULL, 0, 0, "xpt_thrd")) {
-		printf("xpt_init: failed to create rescan thread\n");
-	}
-	/* Install our software interrupt handlers */
-	swi_add(NULL, "cambio", camisr, NULL, SWI_CAMBIO, INTR_MPSAFE, &cambio_ih);
-
 	return (0);
 }
 
@@ -2481,6 +2477,9 @@ xpt_action(union ccb *start_ccb)
 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
 
 	start_ccb->ccb_h.status = CAM_REQ_INPROG;
+	/* Compatibility for RL-unaware code. */
+	if (CAM_PRIORITY_TO_RL(start_ccb->ccb_h.pinfo.priority) == 0)
+	    start_ccb->ccb_h.pinfo.priority += CAM_PRIORITY_NORMAL - 1;
 	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
 }
 
@@ -2546,17 +2545,14 @@ xpt_action_default(union ccb *start_ccb)
 	case XPT_RESET_DEV:
 	case XPT_ENG_EXEC:
 	{
-		struct cam_path *path;
-		int runq;
-
-		path = start_ccb->ccb_h.path;
-
-		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
-		if (path->device->ccbq.queue.qfrozen_cnt == 0)
-			runq = xpt_schedule_dev_sendq(path->bus, path->device);
-		else
-			runq = 0;
-		if (runq != 0)
+		struct cam_path *path = start_ccb->ccb_h.path;
+		int frozen;
+
+		frozen = cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
+		path->device->sim->devq->alloc_openings += frozen;
+		if (frozen > 0)
+			xpt_run_dev_allocq(path->bus);
+		if (xpt_schedule_dev_sendq(path->bus, path->device))
 			xpt_run_dev_sendq(path->bus);
 		break;
 	}
@@ -2601,9 +2597,12 @@ xpt_action_default(union ccb *start_ccb)
 
 			if (abort_ccb->ccb_h.pinfo.index >= 0) {
 				struct cam_ccbq *ccbq;
+				struct cam_ed *device;
 
-				ccbq = &abort_ccb->ccb_h.path->device->ccbq;
-				cam_ccbq_remove_ccb(ccbq, abort_ccb);
+				device = abort_ccb->ccb_h.path->device;
+				ccbq = &device->ccbq;
+				device->sim->devq->alloc_openings -= 
+				    cam_ccbq_remove_ccb(ccbq, abort_ccb);
 				abort_ccb->ccb_h.status =
 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
 				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
@@ -3008,11 +3007,12 @@ xpt_action_default(union ccb *start_ccb)
 		}
 
 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0) {
-
-			xpt_release_devq(crs->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
+			xpt_release_devq_rl(crs->ccb_h.path, /*runlevel*/
+			    (crs->release_flags & RELSIM_RELEASE_RUNLEVEL) ?
+				crs->release_timeout : 0,
+			    /*count*/1, /*run_queue*/TRUE);
 		}
-		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
+		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt[0];
 		start_ccb->ccb_h.status = CAM_REQ_CMP;
 		break;
 	}
@@ -3049,6 +3049,16 @@ xpt_action_default(union ccb *start_ccb)
 #endif /* CAMDEBUG */
 		break;
 	}
+	case XPT_FREEZE_QUEUE:
+	{
+		struct ccb_relsim *crs = &start_ccb->crs;
+
+		xpt_freeze_devq_rl(crs->ccb_h.path, /*runlevel*/
+		    (crs->release_flags & RELSIM_RELEASE_RUNLEVEL) ?
+		    crs->release_timeout : 0, /*count*/1);
+		start_ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
 	case XPT_NOOP:
 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
 			xpt_freeze_devq(start_ccb->ccb_h.path, 1);
@@ -3129,7 +3139,7 @@ void
 xpt_schedule(struct cam_periph *perph, u_int32_t new_priority)
 {
 	struct cam_ed *device;
-	int runq;
+	int runq = 0;
 
 	mtx_assert(perph->sim->mtx, MA_OWNED);
 
@@ -3143,8 +3153,8 @@ xpt_schedule(struct cam_periph *perph, u_int32_t new_priority)
 			camq_change_priority(&device->drvq,
 					     perph->pinfo.index,
 					     new_priority);
+			runq = xpt_schedule_dev_allocq(perph->path->bus, device);
 		}
-		runq = 0;
 	} else {
 		/* New entry on the queue */
 		CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
@@ -3192,8 +3202,9 @@ xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
 					("changed priority to %d\n",
 					 new_priority));
-		}
-		retval = 0;
+			retval = 1;
+		} else
+			retval = 0;
 	} else {
 		/* New entry on the queue */
 		if (new_priority < old_priority)
@@ -3219,15 +3230,15 @@ xpt_run_dev_allocq(struct cam_eb *bus)
 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
 			("   qfrozen_cnt == 0x%x, entries == %d, "
 			 "openings == %d, active == %d\n",
-			 devq->alloc_queue.qfrozen_cnt,
+			 devq->alloc_queue.qfrozen_cnt[0],
 			 devq->alloc_queue.entries,
 			 devq->alloc_openings,
 			 devq->alloc_active));
 
-	devq->alloc_queue.qfrozen_cnt++;
+	devq->alloc_queue.qfrozen_cnt[0]++;
 	while ((devq->alloc_queue.entries > 0)
 	    && (devq->alloc_openings > 0)
-	    && (devq->alloc_queue.qfrozen_cnt <= 1)) {
+	    && (devq->alloc_queue.qfrozen_cnt[0] <= 1)) {
 		struct	cam_ed_qinfo *qinfo;
 		struct	cam_ed *device;
 		union	ccb *work_ccb;
@@ -3237,7 +3248,6 @@ xpt_run_dev_allocq(struct cam_eb *bus)
 		qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->alloc_queue,
 							   CAMQ_HEAD);
 		device = qinfo->device;
-
 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
 				("running device %p\n", device));
 
@@ -3271,15 +3281,13 @@ xpt_run_dev_allocq(struct cam_eb *bus)
 			break;
 		}
 
-		if (drvq->entries > 0) {
-			/* We have more work.  Attempt to reschedule */
-			xpt_schedule_dev_allocq(bus, device);
-		}
+		/* We may have more work. Attempt to reschedule. */
+		xpt_schedule_dev_allocq(bus, device);
 	}
-	devq->alloc_queue.qfrozen_cnt--;
+	devq->alloc_queue.qfrozen_cnt[0]--;
 }
 
-void
+static void
 xpt_run_dev_sendq(struct cam_eb *bus)
 {
 	struct	cam_devq *devq;
@@ -3288,10 +3296,10 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 
 	devq = bus->sim->devq;
 
-	devq->send_queue.qfrozen_cnt++;
+	devq->send_queue.qfrozen_cnt[0]++;
 	while ((devq->send_queue.entries > 0)
 	    && (devq->send_openings > 0)
-	    && (devq->send_queue.qfrozen_cnt <= 1)) {
+	    && (devq->send_queue.qfrozen_cnt[0] <= 1)) {
 		struct	cam_ed_qinfo *qinfo;
 		struct	cam_ed *device;
 		union ccb *work_ccb;
@@ -3300,15 +3308,6 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 		qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->send_queue,
 							   CAMQ_HEAD);
 		device = qinfo->device;
-
-		/*
-		 * If the device has been "frozen", don't attempt
-		 * to run it.
-		 */
-		if (device->ccbq.queue.qfrozen_cnt > 0) {
-			continue;
-		}
-
 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
 				("running device %p\n", device));
 
@@ -3328,7 +3327,7 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 				 * the device queue until we have a slot
 				 * available.
 				 */
-				device->ccbq.queue.qfrozen_cnt++;
+				xpt_freeze_devq(work_ccb->ccb_h.path, 1);
 				STAILQ_INSERT_TAIL(&xsoftc.highpowerq,
 						   &work_ccb->ccb_h,
 						   xpt_links.stqe);
@@ -3350,15 +3349,14 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 		devq->send_openings--;
 		devq->send_active++;
 
-		if (device->ccbq.queue.entries > 0)
-			xpt_schedule_dev_sendq(bus, device);
+		xpt_schedule_dev_sendq(bus, device);
 
 		if (work_ccb && (work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0){
 			/*
 			 * The client wants to freeze the queue
 			 * after this CCB is sent.
 			 */
-			device->ccbq.queue.qfrozen_cnt++;
+			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
 		}
 
 		/* In Target mode, the peripheral driver knows best... */
@@ -3383,7 +3381,7 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 		sim = work_ccb->ccb_h.path->bus->sim;
 		(*(sim->sim_action))(sim, work_ccb);
 	}
-	devq->send_queue.qfrozen_cnt--;
+	devq->send_queue.qfrozen_cnt[0]--;
 }
 
 /*
@@ -3789,13 +3787,9 @@ xpt_release_ccb(union ccb *free_ccb)
 	}
 	sim->devq->alloc_openings++;
 	sim->devq->alloc_active--;
-	/* XXX Turn this into an inline function - xpt_run_device?? */
-	if ((device_is_alloc_queued(device) == 0)
-	 && (device->drvq.entries > 0)) {
+	if (device_is_alloc_queued(device) == 0)
 		xpt_schedule_dev_allocq(bus, device);
-	}
-	if (dev_allocq_is_runnable(sim->devq))
-		xpt_run_dev_allocq(bus);
+	xpt_run_dev_allocq(bus);
 }
 
 /* Functions accessed by SIM drivers */
@@ -3821,7 +3815,7 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
 	struct cam_eb *new_bus;
 	struct cam_eb *old_bus;
 	struct ccb_pathinq cpi;
-	struct cam_path path;
+	struct cam_path *path;
 	cam_status status;
 
 	mtx_assert(sim->mtx, MA_OWNED);
@@ -3833,6 +3827,11 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
 		/* Couldn't satisfy request */
 		return (CAM_RESRC_UNAVAIL);
 	}
+	path = (struct cam_path *)malloc(sizeof(*path), M_CAMXPT, M_NOWAIT);
+	if (path == NULL) {
+		free(new_bus, M_CAMXPT);
+		return (CAM_RESRC_UNAVAIL);
+	}
 
 	if (strcmp(sim->sim_name, "xpt") != 0) {
 		sim->path_id =
@@ -3867,13 +3866,12 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
 	 */
 	new_bus->xport = &xport_default;
 
-	bzero(&path, sizeof(path));
-	status = xpt_compile_path(&path, /*periph*/NULL, sim->path_id,
+	status = xpt_compile_path(path, /*periph*/NULL, sim->path_id,
 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
 	if (status != CAM_REQ_CMP)
 		printf("xpt_compile_path returned %d\n", status);
 
-	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
+	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
 	cpi.ccb_h.func_code = XPT_PATH_INQ;
 	xpt_action((union ccb *)&cpi);
 
@@ -3899,9 +3897,17 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
 
 	/* Notify interested parties */
 	if (sim->path_id != CAM_XPT_PATH_ID) {
-		xpt_async(AC_PATH_REGISTERED, &path, &cpi);
-	}
-	xpt_release_path(&path);
+		union	ccb *scan_ccb;
+
+		xpt_async(AC_PATH_REGISTERED, path, &cpi);
+		/* Initiate bus rescan. */
+		scan_ccb = xpt_alloc_ccb_nowait();
+		scan_ccb->ccb_h.path = path;
+		scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
+		scan_ccb->crcn.flags = 0;
+		xpt_rescan(scan_ccb);
+	} else
+		xpt_free_path(path);
 	return (CAM_SUCCESS);
 }
 
@@ -4110,12 +4116,34 @@ xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
 }
 
 u_int32_t
-xpt_freeze_devq(struct cam_path *path, u_int count)
+xpt_freeze_devq_rl(struct cam_path *path, cam_rl rl, u_int count)
 {
+	struct cam_ed *dev = path->device;
 
 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
-	path->device->ccbq.queue.qfrozen_cnt += count;
-	return (path->device->ccbq.queue.qfrozen_cnt);
+	dev->sim->devq->alloc_openings +=
+	    cam_ccbq_freeze(&dev->ccbq, rl, count);
+	/* Remove frozen device from allocq. */
+	if (device_is_alloc_queued(dev) &&
+	    cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
+	     CAMQ_GET_PRIO(&dev->drvq)))) {
+		camq_remove(&dev->sim->devq->alloc_queue,
+		    dev->alloc_ccb_entry.pinfo.index);
+	}
+	/* Remove frozen device from sendq. */
+	if (device_is_send_queued(dev) &&
+	    cam_ccbq_frozen_top(&dev->ccbq)) {
+		camq_remove(&dev->sim->devq->send_queue,
+		    dev->send_ccb_entry.pinfo.index);
+	}
+	return (dev->ccbq.queue.qfrozen_cnt[rl]);
+}
+
+u_int32_t
+xpt_freeze_devq(struct cam_path *path, u_int count)
+{
+
+	return (xpt_freeze_devq_rl(path, 0, count));
 }
 
 u_int32_t
@@ -4123,8 +4151,8 @@ xpt_freeze_simq(struct cam_sim *sim, u_int count)
 {
 
 	mtx_assert(sim->mtx, MA_OWNED);
-	sim->devq->send_queue.qfrozen_cnt += count;
-	return (sim->devq->send_queue.qfrozen_cnt);
+	sim->devq->send_queue.qfrozen_cnt[0] += count;
+	return (sim->devq->send_queue.qfrozen_cnt[0]);
 }
 
 static void
@@ -4134,7 +4162,7 @@ xpt_release_devq_timeout(void *arg)
 
 	device = (struct cam_ed *)arg;
 
-	xpt_release_devq_device(device, /*count*/1, /*run_queue*/TRUE);
+	xpt_release_devq_device(device, /*rl*/0, /*count*/1, /*run_queue*/TRUE);
 }
 
 void
@@ -4142,51 +4170,59 @@ xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
 {
 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
-	xpt_release_devq_device(path->device, count, run_queue);
+	xpt_release_devq_device(path->device, /*rl*/0, count, run_queue);
 }
 
-static void
-xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
+void
+xpt_release_devq_rl(struct cam_path *path, cam_rl rl, u_int count, int run_queue)
 {
-	int	rundevq;
-
-	rundevq = 0;
-	if (dev->ccbq.queue.qfrozen_cnt > 0) {
-
-		count = (count > dev->ccbq.queue.qfrozen_cnt) ?
-		    dev->ccbq.queue.qfrozen_cnt : count;
-		dev->ccbq.queue.qfrozen_cnt -= count;
-		if (dev->ccbq.queue.qfrozen_cnt == 0) {
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
-			/*
-			 * No longer need to wait for a successful
-			 * command completion.
-			 */
-			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
+	xpt_release_devq_device(path->device, rl, count, run_queue);
+}
 
-			/*
-			 * Remove any timeouts that might be scheduled
-			 * to release this queue.
-			 */
-			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
-				callout_stop(&dev->callout);
-				dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
-			}
+static void
+xpt_release_devq_device(struct cam_ed *dev, cam_rl rl, u_int count, int run_queue)
+{
 
-			/*
-			 * Now that we are unfrozen schedule the
-			 * device so any pending transactions are
-			 * run.
-			 */
-			if ((dev->ccbq.queue.entries > 0)
-			 && (xpt_schedule_dev_sendq(dev->target->bus, dev))
-			 && (run_queue != 0)) {
-				rundevq = 1;
-			}
+	if (count > dev->ccbq.queue.qfrozen_cnt[rl]) {
+#ifdef INVARIANTS
+		printf("xpt_release_devq(%d): requested %u > present %u\n",
+		    rl, count, dev->ccbq.queue.qfrozen_cnt[rl]);
+#endif
+		count = dev->ccbq.queue.qfrozen_cnt[rl];
+	}
+	dev->sim->devq->alloc_openings -=
+	    cam_ccbq_release(&dev->ccbq, rl, count);
+	if (cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
+	    CAMQ_GET_PRIO(&dev->drvq))) == 0) {
+		if (xpt_schedule_dev_allocq(dev->target->bus, dev))
+			xpt_run_dev_allocq(dev->target->bus);
+	}
+	if (cam_ccbq_frozen_top(&dev->ccbq) == 0) {
+		/*
+		 * No longer need to wait for a successful
+		 * command completion.
+		 */
+		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
+		/*
+		 * Remove any timeouts that might be scheduled
+		 * to release this queue.
+		 */
+		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
+			callout_stop(&dev->callout);
+			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
 		}
+		if (run_queue == 0)
+			return;
+		/*
+		 * Now that we are unfrozen schedule the
+		 * device so any pending transactions are
+		 * run.
+		 */
+		if (xpt_schedule_dev_sendq(dev->target->bus, dev))
+			xpt_run_dev_sendq(dev->target->bus);
 	}
-	if (rundevq != 0)
-		xpt_run_dev_sendq(dev->target->bus);
 }
 
 void
@@ -4195,32 +4231,33 @@ xpt_release_simq(struct cam_sim *sim, int run_queue)
 	struct	camq *sendq;
 
 	mtx_assert(sim->mtx, MA_OWNED);
-
 	sendq = &(sim->devq->send_queue);
-	if (sendq->qfrozen_cnt > 0) {
+	if (sendq->qfrozen_cnt[0] <= 0) {
+#ifdef INVARIANTS
+		printf("xpt_release_simq: requested 1 > present %u\n",
+		    sendq->qfrozen_cnt[0]);
+#endif
+	} else
+		sendq->qfrozen_cnt[0]--;
+	if (sendq->qfrozen_cnt[0] == 0) {
+		/*
+		 * If there is a timeout scheduled to release this
+		 * sim queue, remove it.  The queue frozen count is
+		 * already at 0.
+		 */
+		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
+			callout_stop(&sim->callout);
+			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
+		}
+		if (run_queue) {
+			struct cam_eb *bus;
 
-		sendq->qfrozen_cnt--;
-		if (sendq->qfrozen_cnt == 0) {
 			/*
-			 * If there is a timeout scheduled to release this
-			 * sim queue, remove it.  The queue frozen count is
-			 * already at 0.
+			 * Now that we are unfrozen run the send queue.
 			 */
-			if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
-				callout_stop(&sim->callout);
-				sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
-			}
-
-			if (run_queue) {
-				struct cam_eb *bus;
-
-				/*
-				 * Now that we are unfrozen run the send queue.
-				 */
-				bus = xpt_find_bus(sim->path_id);
-				xpt_run_dev_sendq(bus);
-				xpt_release_bus(bus);
-			}
+			bus = xpt_find_bus(sim->path_id);
+			xpt_run_dev_sendq(bus);
+			xpt_release_bus(bus);
 		}
 	}
 }
@@ -4399,7 +4436,7 @@ xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
 
 	device->mintags = 1;
 	device->maxtags = 1;
-	bus->sim->max_ccbs = device->ccbq.devq_openings;
+	bus->sim->max_ccbs += device->ccbq.devq_openings;
 	cur_device = TAILQ_FIRST(&target->ed_entries);
 	while (cur_device != NULL && cur_device->lun_id < lun_id)
 		cur_device = TAILQ_NEXT(cur_device, links);
@@ -4629,104 +4666,16 @@ xpt_stop_tags(struct cam_path *path)
 	xpt_action((union ccb *)&crs);
 }
 
-static int busses_to_config;
-static int busses_to_reset;
-
-static int
-xptconfigbuscountfunc(struct cam_eb *bus, void *arg)
-{
-
-	mtx_assert(bus->sim->mtx, MA_OWNED);
-
-	if (bus->path_id != CAM_XPT_PATH_ID) {
-		struct cam_path path;
-		struct ccb_pathinq cpi;
-		int can_negotiate;
-
-		busses_to_config++;
-		xpt_compile_path(&path, NULL, bus->path_id,
-				 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
-		xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
-		cpi.ccb_h.func_code = XPT_PATH_INQ;
-		xpt_action((union ccb *)&cpi);
-		can_negotiate = cpi.hba_inquiry;
-		can_negotiate &= (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE);
-		if ((cpi.hba_misc & PIM_NOBUSRESET) == 0
-		 && can_negotiate)
-			busses_to_reset++;
-		xpt_release_path(&path);
-	}
-
-	return(1);
-}
-
-static int
-xptconfigfunc(struct cam_eb *bus, void *arg)
+static void
+xpt_boot_delay(void *arg)
 {
-	struct	cam_path *path;
-	union	ccb *work_ccb;
 
-	mtx_assert(bus->sim->mtx, MA_OWNED);
-
-	if (bus->path_id != CAM_XPT_PATH_ID) {
-		cam_status status;
-		int can_negotiate;
-
-		work_ccb = xpt_alloc_ccb_nowait();
-		if (work_ccb == NULL) {
-			busses_to_config--;
-			xpt_finishconfig(xpt_periph, NULL);
-			return(0);
-		}
-		if ((status = xpt_create_path(&path, xpt_periph, bus->path_id,
-					      CAM_TARGET_WILDCARD,
-					      CAM_LUN_WILDCARD)) !=CAM_REQ_CMP){
-			printf("xptconfigfunc: xpt_create_path failed with "
-			       "status %#x for scbus%d\n", status, bus->path_id);
-			printf("xptconfigfunc: halting bus configuration\n");
-			xpt_free_ccb(work_ccb);
-			busses_to_config--;
-			xpt_finishconfig(xpt_periph, NULL);
-			return(0);
-		}
-		xpt_setup_ccb(&work_ccb->ccb_h, path, CAM_PRIORITY_NORMAL);
-		work_ccb->ccb_h.func_code = XPT_PATH_INQ;
-		xpt_action(work_ccb);
-		if (work_ccb->ccb_h.status != CAM_REQ_CMP) {
-			printf("xptconfigfunc: CPI failed on scbus%d "
-			       "with status %d\n", bus->path_id,
-			       work_ccb->ccb_h.status);
-			xpt_finishconfig(xpt_periph, work_ccb);
-			return(1);
-		}
-
-		can_negotiate = work_ccb->cpi.hba_inquiry;
-		can_negotiate &= (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE);
-		if ((work_ccb->cpi.hba_misc & PIM_NOBUSRESET) == 0
-		 && (can_negotiate != 0)) {
-			xpt_setup_ccb(&work_ccb->ccb_h, path, CAM_PRIORITY_NORMAL);
-			work_ccb->ccb_h.func_code = XPT_RESET_BUS;
-			work_ccb->ccb_h.cbfcnp = NULL;
-			CAM_DEBUG(path, CAM_DEBUG_SUBTRACE,
-				  ("Resetting Bus\n"));
-			xpt_action(work_ccb);
-			xpt_finishconfig(xpt_periph, work_ccb);
-		} else {
-			/* Act as though we performed a successful BUS RESET */
-			work_ccb->ccb_h.func_code = XPT_RESET_BUS;
-			xpt_finishconfig(xpt_periph, work_ccb);
-		}
-	}
-
-	return(1);
+	xpt_release_boot();
 }
 
 static void
 xpt_config(void *arg)
 {
-	struct	periph_driver **p_drv;
-	int	i;
-
 	/*
 	 * Now that interrupts are enabled, go find our devices
 	 */
@@ -4760,28 +4709,43 @@ xpt_config(void *arg)
 #endif /* CAM_DEBUG_BUS */
 #endif /* CAMDEBUG */
 
-	/* Register early peripheral drivers */
-	/* XXX This will have to change when we have loadable modules */
-	p_drv = periph_drivers;
-	for (i = 0; p_drv[i] != NULL; i++) {
-		if ((p_drv[i]->flags & CAM_PERIPH_DRV_EARLY) != 0)
-			(*p_drv[i]->init)();
+	periphdriver_init(1);
+	xpt_hold_boot();
+	callout_init(&xsoftc.boot_callout, 1);
+	callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000,
+	    xpt_boot_delay, NULL);
+	/* Fire up rescan thread. */
+	if (kproc_create(xpt_scanner_thread, NULL, NULL, 0, 0, "xpt_thrd")) {
+		printf("xpt_init: failed to create rescan thread\n");
 	}
-	/*
-	 * Scan all installed busses.
-	 */
-	xpt_for_all_busses(xptconfigbuscountfunc, NULL);
+}
+
+void
+xpt_hold_boot(void)
+{
+	xpt_lock_buses();
+	xsoftc.buses_to_config++;
+	xpt_unlock_buses();
+}
+
+void
+xpt_release_boot(void)
+{
+	xpt_lock_buses();
+	xsoftc.buses_to_config--;
+	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
+		struct	xpt_task *task;
 
-	if (busses_to_config == 0) {
+		xsoftc.buses_config_done = 1;
+		xpt_unlock_buses();
 		/* Call manually because we don't have any busses */
-		xpt_finishconfig(xpt_periph, NULL);
-	} else  {
-		if (busses_to_reset > 0 && scsi_delay >= 2000) {
-			printf("Waiting %d seconds for SCSI "
-			       "devices to settle\n", scsi_delay/1000);
+		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
+		if (task != NULL) {
+			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
+			taskqueue_enqueue(taskqueue_thread, &task->task);
 		}
-		xpt_for_all_busses(xptconfigfunc, NULL);
-	}
+	} else
+		xpt_unlock_buses();
 }
 
 /*
@@ -4809,72 +4773,23 @@ xptpassannouncefunc(struct cam_ed *device, void *arg)
 static void
 xpt_finishconfig_task(void *context, int pending)
 {
-	struct	periph_driver **p_drv;
-	int	i;
-
-	if (busses_to_config == 0) {
-		/* Register all the peripheral drivers */
-		/* XXX This will have to change when we have loadable modules */
-		p_drv = periph_drivers;
-		for (i = 0; p_drv[i] != NULL; i++) {
-			if ((p_drv[i]->flags & CAM_PERIPH_DRV_EARLY) == 0)
-				(*p_drv[i]->init)();
-		}
 
-		/*
-		 * Check for devices with no "standard" peripheral driver
-		 * attached.  For any devices like that, announce the
-		 * passthrough driver so the user will see something.
-		 */
-		xpt_for_all_devices(xptpassannouncefunc, NULL);
+	periphdriver_init(2);
+	/*
+	 * Check for devices with no "standard" peripheral driver
+	 * attached.  For any devices like that, announce the
+	 * passthrough driver so the user will see something.
+	 */
+	xpt_for_all_devices(xptpassannouncefunc, NULL);
 
-		/* Release our hook so that the boot can continue. */
-		config_intrhook_disestablish(xsoftc.xpt_config_hook);
-		free(xsoftc.xpt_config_hook, M_CAMXPT);
-		xsoftc.xpt_config_hook = NULL;
-	}
+	/* Release our hook so that the boot can continue. */
+	config_intrhook_disestablish(xsoftc.xpt_config_hook);
+	free(xsoftc.xpt_config_hook, M_CAMXPT);
+	xsoftc.xpt_config_hook = NULL;
 
 	free(context, M_CAMXPT);
 }
 
-static void
-xpt_finishconfig(struct cam_periph *periph, union ccb *done_ccb)
-{
-	struct	xpt_task *task;
-
-	if (done_ccb != NULL) {
-		CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
-			  ("xpt_finishconfig\n"));
-		switch(done_ccb->ccb_h.func_code) {
-		case XPT_RESET_BUS:
-			if (done_ccb->ccb_h.status == CAM_REQ_CMP) {
-				done_ccb->ccb_h.func_code = XPT_SCAN_BUS;
-				done_ccb->ccb_h.cbfcnp = xpt_finishconfig;
-				done_ccb->crcn.flags = 0;
-				xpt_action(done_ccb);
-				return;
-			}
-			/* FALLTHROUGH */
-		case XPT_SCAN_BUS:
-		default:
-			xpt_free_path(done_ccb->ccb_h.path);
-			busses_to_config--;
-			break;
-		}
-	}
-
-	if (busses_to_config == 0) {
-		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
-		if (task != NULL) {
-			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
-			taskqueue_enqueue(taskqueue_thread, &task->task);
-		}
-	}
-
-	if (done_ccb != NULL)
-		xpt_free_ccb(done_ccb);
-}
-
 cam_status
 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
 		   struct cam_path *path)
@@ -4894,7 +4809,7 @@ xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
 		xptpath = 1;
 	}
 
-	xpt_setup_ccb(&csa.ccb_h, path, /*priority*/5);
+	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
 	csa.ccb_h.func_code = XPT_SASYNC_CB;
 	csa.event_enable = event;
 	csa.callback = cbfunc;
@@ -5047,26 +4962,19 @@ camisr_runqueue(void *V_queue)
 			cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
 			ccb_h->path->bus->sim->devq->send_active--;
 			ccb_h->path->bus->sim->devq->send_openings++;
+			runq = TRUE;
 
 			if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
 			  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)
 			 || ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
 			  && (dev->ccbq.dev_active == 0))) {
-
 				xpt_release_devq(ccb_h->path, /*count*/1,
-						 /*run_queue*/TRUE);
+						 /*run_queue*/FALSE);
 			}
 
 			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
 			 && (--dev->tag_delay_count == 0))
 				xpt_start_tags(ccb_h->path);
-
-			if ((dev->ccbq.queue.entries > 0)
-			 && (dev->ccbq.queue.qfrozen_cnt == 0)
-			 && (device_is_send_queued(dev) == 0)) {
-				runq = xpt_schedule_dev_sendq(ccb_h->path->bus,
-							      dev);
-			}
 		}
 
 		if (ccb_h->status & CAM_RELEASE_SIMQ) {