Refactor CTL datamove KPI.

 - Make frontends call unified CTL core method ctl_datamove_done()
to report move completion.  It allows to reduce code duplication
in differerent backends by accounting DMA time in common code.
 - Add to ctl_datamove_done() and be_move_done() callback samethr
argument, reporting whether the callback is called in the same
context as ctl_datamove().  It allows for some cases like iSCSI
write with immediate data or camsim frontend write save one context
switch, since we know that the context is sleepable.
 - Remove data_move_done() methods from struct ctl_backend_driver,
unused since forever.

MFC after:	 1 month

(cherry picked from commit 2c7dc6bae9fd5c2fa0a65768df8e4e99c2f159f1)

12 files changed, 94 insertions, 162 deletions

diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index b9d13a2e8c5d..07331091fc54 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -534,9 +534,9 @@ static void ctl_done_timer_wakeup(void *arg);
 
 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
-static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
+static int ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr);
 static void ctl_datamove_remote_write(union ctl_io *io);
-static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
+static int ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr);
 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
@@ -736,7 +736,7 @@ ctl_ha_datamove(union ctl_io *io)
 		    sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries,
 		    M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
 			io->io_hdr.port_status = 31341;
-			io->scsiio.be_move_done(io);
+			ctl_datamove_done(io, true);
 			return;
 		}
 		msg.dt.sent_sg_entries = sg_entries_sent;
@@ -753,7 +753,7 @@ ctl_ha_datamove(union ctl_io *io)
 		if (lun)
 			mtx_unlock(&lun->lun_lock);
 		io->io_hdr.port_status = 31342;
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done(io, true);
 		return;
 	}
 	io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
@@ -5032,7 +5032,7 @@ ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
  * make it down to say RAIDCore's configuration code.
  */
 int
-ctl_config_move_done(union ctl_io *io)
+ctl_config_move_done(union ctl_io *io, bool samethr)
 {
 	int retval;
 
@@ -5040,18 +5040,6 @@ ctl_config_move_done(union ctl_io *io)
 	KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
 	    ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
 
-	if ((io->io_hdr.port_status != 0) &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
-	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
-		ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
-		    /*retry_count*/ io->io_hdr.port_status);
-	} else if (io->scsiio.kern_data_resid != 0 &&
-	    (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
-	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
-		ctl_set_invalid_field_ciu(&io->scsiio);
-	}
-
 	if (ctl_debug & CTL_DEBUG_CDB_DATA)
 		ctl_data_print(io);
 	if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
@@ -12314,7 +12302,7 @@ ctl_handle_isc(union ctl_io *io)
 		ctl_datamove_remote(io);
 		break;
 	case CTL_MSG_DATAMOVE_DONE:	/* Only used in XFER mode */
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done(io, false);
 		break;
 	case CTL_MSG_FAILOVER:
 		ctl_failover_lun(io);
@@ -12474,6 +12462,45 @@ ctl_datamove_timer_wakeup(void *arg)
 }
 #endif /* CTL_IO_DELAY */
 
+static void
+ctl_datamove_done_process(union ctl_io *io)
+{
+#ifdef CTL_TIME_IO
+	struct bintime cur_bt;
+#endif
+
+	KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
+	    ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
+
+#ifdef CTL_TIME_IO
+	getbinuptime(&cur_bt);
+	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
+	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
+#endif
+	io->io_hdr.num_dmas++;
+
+	if ((io->io_hdr.port_status != 0) &&
+	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
+	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
+		ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
+		    /*retry_count*/ io->io_hdr.port_status);
+	} else if (io->scsiio.kern_data_resid != 0 &&
+	    (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
+	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
+	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
+		ctl_set_invalid_field_ciu(&io->scsiio);
+	} else if (ctl_debug & CTL_DEBUG_CDB_DATA)
+		ctl_data_print(io);
+}
+
+void
+ctl_datamove_done(union ctl_io *io, bool samethr)
+{
+
+	ctl_datamove_done_process(io);
+	io->scsiio.be_move_done(io, samethr);
+}
+
 void
 ctl_datamove(union ctl_io *io)
 {
@@ -12487,39 +12514,7 @@ ctl_datamove(union ctl_io *io)
 	io->scsiio.kern_data_resid = io->scsiio.kern_data_len;
 
 #ifdef CTL_TIME_IO
-	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
-		char str[256];
-		char path_str[64];
-		struct sbuf sb;
-
-		ctl_scsi_path_string(io, path_str, sizeof(path_str));
-		sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
-
-		sbuf_cat(&sb, path_str);
-		switch (io->io_hdr.io_type) {
-		case CTL_IO_SCSI:
-			ctl_scsi_command_string(&io->scsiio, NULL, &sb);
-			sbuf_printf(&sb, "\n");
-			sbuf_cat(&sb, path_str);
-			sbuf_printf(&sb, "Tag: 0x%04x/%d, Prio: %d\n",
-				    io->scsiio.tag_num, io->scsiio.tag_type,
-				    io->scsiio.priority);
-			break;
-		case CTL_IO_TASK:
-			sbuf_printf(&sb, "Task Action: %d Tag: 0x%04x/%d\n",
-				    io->taskio.task_action,
-				    io->taskio.tag_num, io->taskio.tag_type);
-			break;
-		default:
-			panic("%s: Invalid CTL I/O type %d\n",
-			    __func__, io->io_hdr.io_type);
-		}
-		sbuf_cat(&sb, path_str);
-		sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
-			    (intmax_t)time_uptime - io->io_hdr.start_time);
-		sbuf_finish(&sb);
-		printf("%s", sbuf_data(&sb));
-	}
+	getbinuptime(&io->io_hdr.dma_start_bt);
 #endif /* CTL_TIME_IO */
 
 #ifdef CTL_IO_DELAY
@@ -12555,18 +12550,15 @@ ctl_datamove(union ctl_io *io)
 		       io->io_hdr.nexus.targ_port,
 		       io->io_hdr.nexus.targ_lun);
 		io->io_hdr.port_status = 31337;
-		/*
-		 * Note that the backend, in this case, will get the
-		 * callback in its context.  In other cases it may get
-		 * called in the frontend's interrupt thread context.
-		 */
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done_process(io);
+		io->scsiio.be_move_done(io, true);
 		return;
 	}
 
 	/* Don't confuse frontend with zero length data move. */
 	if (io->scsiio.kern_data_len == 0) {
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done_process(io);
+		io->scsiio.be_move_done(io, true);
 		return;
 	}
 
@@ -12653,7 +12645,7 @@ ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
  * need to push it over to the remote controller's memory.
  */
 static int
-ctl_datamove_remote_dm_write_cb(union ctl_io *io)
+ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr)
 {
 	int retval;
 
@@ -12692,7 +12684,7 @@ ctl_datamove_remote_write(union ctl_io *io)
 }
 
 static int
-ctl_datamove_remote_dm_read_cb(union ctl_io *io)
+ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr)
 {
 	uint32_t i;
 
diff --git a/sys/cam/ctl/ctl.h b/sys/cam/ctl/ctl.h
index 56dd5313b4cb..be3e4a37b157 100644
--- a/sys/cam/ctl/ctl.h
+++ b/sys/cam/ctl/ctl.h
@@ -170,7 +170,8 @@ int ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
 int ctl_ie_log_sense_handler(struct ctl_scsiio *ctsio,
 				   struct ctl_page_index *page_index,
 				   int pc);
-int ctl_config_move_done(union ctl_io *io);
+int ctl_config_move_done(union ctl_io *io, bool samethr);
+void ctl_datamove_done(union ctl_io *io, bool samethr);
 void ctl_datamove(union ctl_io *io);
 void ctl_serseq_done(union ctl_io *io);
 void ctl_done(union ctl_io *io);
diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h
index 4d75a20f3559..35113f4750d8 100644
--- a/sys/cam/ctl/ctl_backend.h
+++ b/sys/cam/ctl/ctl_backend.h
@@ -187,7 +187,6 @@ struct ctl_backend_driver {
 	be_init_t	  init;			 /* passed to CTL */
 	be_shutdown_t	  shutdown;		 /* passed to CTL */
 	be_func_t	  data_submit;		 /* passed to CTL */
-	be_func_t	  data_move_done;	 /* passed to CTL */
 	be_func_t	  config_read;		 /* passed to CTL */
 	be_func_t	  config_write;		 /* passed to CTL */
 	be_ioctl_t	  ioctl;		 /* passed to CTL */
diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c
index 2ed3cf32abe9..abcd8d6e6fae 100644
--- a/sys/cam/ctl/ctl_backend_block.c
+++ b/sys/cam/ctl/ctl_backend_block.c
@@ -234,7 +234,7 @@ SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
 static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
 static void ctl_free_beio(struct ctl_be_block_io *beio);
 static void ctl_complete_beio(struct ctl_be_block_io *beio);
-static int ctl_be_block_move_done(union ctl_io *io);
+static int ctl_be_block_move_done(union ctl_io *io, bool samethr);
 static void ctl_be_block_biodone(struct bio *bio);
 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 				    struct ctl_be_block_io *beio);
@@ -290,7 +290,6 @@ static struct ctl_backend_driver ctl_be_block_driver =
 	.init = ctl_be_block_init,
 	.shutdown = ctl_be_block_shutdown,
 	.data_submit = ctl_be_block_submit,
-	.data_move_done = ctl_be_block_move_done,
 	.config_read = ctl_be_block_config_read,
 	.config_write = ctl_be_block_config_write,
 	.ioctl = ctl_be_block_ioctl,
@@ -412,46 +411,23 @@ ctl_be_block_compare(union ctl_io *io)
 }
 
 static int
-ctl_be_block_move_done(union ctl_io *io)
+ctl_be_block_move_done(union ctl_io *io, bool samethr)
 {
 	struct ctl_be_block_io *beio;
 	struct ctl_be_block_lun *be_lun;
 	struct ctl_lba_len_flags *lbalen;
-#ifdef CTL_TIME_IO
-	struct bintime cur_bt;
-#endif
 
 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 	be_lun = beio->lun;
 
 	DPRINTF("entered\n");
-
-#ifdef CTL_TIME_IO
-	getbinuptime(&cur_bt);
-	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
-	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
-#endif
-	io->io_hdr.num_dmas++;
 	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 
 	/*
-	 * We set status at this point for read commands, and write
-	 * commands with errors.
+	 * We set status at this point for read and compare commands.
 	 */
-	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
-		;
-	} else if ((io->io_hdr.port_status != 0) &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
-	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
-		ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
-		    /*retry_count*/ io->io_hdr.port_status);
-	} else if (io->scsiio.kern_data_resid != 0 &&
-	    (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
-	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
-		ctl_set_invalid_field_ciu(&io->scsiio);
-	} else if ((io->io_hdr.port_status == 0) &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
+	if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
+	    (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
 		lbalen = ARGS(beio->io);
 		if (lbalen->flags & CTL_LLF_READ) {
 			ctl_set_success(&io->scsiio);
@@ -472,18 +448,22 @@ ctl_be_block_move_done(union ctl_io *io)
 	}
 
 	/*
-	 * At this point, we have a write and the DMA completed
-	 * successfully.  We now have to queue it to the task queue to
+	 * At this point, we have a write and the DMA completed successfully.
+	 * If we were called synchronously in the original thread then just
+	 * dispatch, otherwise we now have to queue it to the task queue to
 	 * execute the backend I/O.  That is because we do blocking
 	 * memory allocations, and in the file backing case, blocking I/O.
 	 * This move done routine is generally called in the SIM's
 	 * interrupt context, and therefore we cannot block.
 	 */
-	mtx_lock(&be_lun->queue_lock);
-	STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
-	mtx_unlock(&be_lun->queue_lock);
-	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
-
+	if (samethr) {
+		be_lun->dispatch(be_lun, beio);
+	} else {
+		mtx_lock(&be_lun->queue_lock);
+		STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
+		mtx_unlock(&be_lun->queue_lock);
+		taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
+	}
 	return (0);
 }
 
@@ -578,9 +558,6 @@ ctl_be_block_biodone(struct bio *bio)
 			ctl_set_success(&io->scsiio);
 			ctl_serseq_done(io);
 		}
-#ifdef CTL_TIME_IO
-		getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
 		ctl_datamove(io);
 	}
 }
@@ -795,9 +772,6 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 			ctl_set_success(&io->scsiio);
 			ctl_serseq_done(io);
 		}
-#ifdef CTL_TIME_IO
-		getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
 		ctl_datamove(io);
 	}
 }
@@ -966,9 +940,6 @@ ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 			ctl_set_success(&io->scsiio);
 			ctl_serseq_done(io);
 		}
-#ifdef CTL_TIME_IO
-		getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
 		ctl_datamove(io);
 	}
 }
@@ -1663,9 +1634,6 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 		be_lun->dispatch(be_lun, beio);
 	} else {
 		SDT_PROBE0(cbb, , write, alloc_done);
-#ifdef CTL_TIME_IO
-		getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
 		ctl_datamove(io);
 	}
 }
diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c
index 2595aa0be00e..e67d699bda70 100644
--- a/sys/cam/ctl/ctl_backend_ramdisk.c
+++ b/sys/cam/ctl/ctl_backend_ramdisk.c
@@ -139,7 +139,7 @@ extern struct ctl_softc *control_softc;
 
 static int ctl_backend_ramdisk_init(void);
 static int ctl_backend_ramdisk_shutdown(void);
-static int ctl_backend_ramdisk_move_done(union ctl_io *io);
+static int ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr);
 static void ctl_backend_ramdisk_compare(union ctl_io *io);
 static void ctl_backend_ramdisk_rw(union ctl_io *io);
 static int ctl_backend_ramdisk_submit(union ctl_io *io);
@@ -164,7 +164,6 @@ static struct ctl_backend_driver ctl_be_ramdisk_driver =
 	.init = ctl_backend_ramdisk_init,
 	.shutdown = ctl_backend_ramdisk_shutdown,
 	.data_submit = ctl_backend_ramdisk_submit,
-	.data_move_done = ctl_backend_ramdisk_move_done,
 	.config_read = ctl_backend_ramdisk_config_read,
 	.config_write = ctl_backend_ramdisk_config_write,
 	.ioctl = ctl_backend_ramdisk_ioctl,
@@ -402,38 +401,17 @@ ctl_backend_ramdisk_cmp(union ctl_io *io)
 }
 
 static int
-ctl_backend_ramdisk_move_done(union ctl_io *io)
+ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr)
 {
 	struct ctl_be_ramdisk_lun *be_lun =
 	    (struct ctl_be_ramdisk_lun *)CTL_BACKEND_LUN(io);
-#ifdef CTL_TIME_IO
-	struct bintime cur_bt;
-#endif
 
 	CTL_DEBUG_PRINT(("ctl_backend_ramdisk_move_done\n"));
-#ifdef CTL_TIME_IO
-	getbinuptime(&cur_bt);
-	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
-	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
-#endif
-	io->io_hdr.num_dmas++;
 	if (io->scsiio.kern_sg_entries > 0)
 		free(io->scsiio.kern_data_ptr, M_RAMDISK);
 	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
-	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
-		;
-	} else if (io->io_hdr.port_status != 0 &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
-	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
-		ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
-		    /*retry_count*/ io->io_hdr.port_status);
-	} else if (io->scsiio.kern_data_resid != 0 &&
-	    (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
-	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
-		ctl_set_invalid_field_ciu(&io->scsiio);
-	} else if ((io->io_hdr.port_status == 0) &&
-	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
+	if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
+	    (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
 		if (ARGS(io)->flags & CTL_LLF_COMPARE) {
 			/* We have data block ready for comparison. */
 			if (ctl_backend_ramdisk_cmp(io))
@@ -471,9 +449,6 @@ ctl_backend_ramdisk_compare(union ctl_io *io)
 	io->scsiio.kern_sg_entries = 0;
 	io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 	PRIV(io)->len += lbas;
-#ifdef CTL_TIME_IO
-	getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
 	ctl_datamove(io);
 }
 
@@ -534,9 +509,6 @@ nospc:
 		ctl_set_success(&io->scsiio);
 		ctl_serseq_done(io);
 	}
-#ifdef CTL_TIME_IO
-	getbinuptime(&io->io_hdr.dma_start_bt);
-#endif
 	ctl_datamove(io);
 }
 
diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c
index 0f40760f1dc1..3afe68cc7bde 100644
--- a/sys/cam/ctl/ctl_frontend_cam_sim.c
+++ b/sys/cam/ctl/ctl_frontend_cam_sim.c
@@ -416,7 +416,7 @@ cfcs_datamove(union ctl_io *io)
 		xpt_done(ccb);
 	}
 
-	io->scsiio.be_move_done(io);
+	ctl_datamove_done(io, true);
 }
 
 static void
diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c
index b0298728e8db..f2403353a060 100644
--- a/sys/cam/ctl/ctl_frontend_ioctl.c
+++ b/sys/cam/ctl/ctl_frontend_ioctl.c
@@ -568,7 +568,7 @@ cfi_submit_wait(union ctl_io *io)
 			 * will immediately call back and wake us up,
 			 * probably using our own context.
 			 */
-			io->scsiio.be_move_done(io);
+			ctl_datamove_done(io, false);
 			break;
 		case CTL_IOCTL_DONE:
 			mtx_unlock(&params.ioctl_mtx);
diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index 4ad238ea6909..82ab2fc3b8c6 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -923,7 +923,7 @@ cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 		cfiscsi_data_wait_free(cs, cdw);
 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 		if (done)
-			io->scsiio.be_move_done(io);
+			ctl_datamove_done(io, false);
 		else
 			cfiscsi_datamove_out(io);
 	}
@@ -1136,7 +1136,7 @@ cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
 		 */
 		cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
-		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
+		ctl_datamove_done(cdw->cdw_ctl_io, false);
 		cfiscsi_data_wait_free(cs, cdw);
 		CFISCSI_SESSION_LOCK(cs);
 	}
@@ -2469,7 +2469,7 @@ cfiscsi_datamove_in(union ctl_io *io)
 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
 		    "already sent the expected len", buffer_offset);
 #endif
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done(io, true);
 		return;
 	}
 
@@ -2485,7 +2485,7 @@ cfiscsi_datamove_in(union ctl_io *io)
 				CFISCSI_SESSION_WARN(cs, "failed to "
 				    "allocate memory; dropping connection");
 				ctl_set_busy(&io->scsiio);
-				io->scsiio.be_move_done(io);
+				ctl_datamove_done(io, true);
 				cfiscsi_session_terminate(cs);
 				return;
 			}
@@ -2542,7 +2542,7 @@ cfiscsi_datamove_in(union ctl_io *io)
 			    "allocate memory; dropping connection");
 			icl_pdu_free(response);
 			ctl_set_busy(&io->scsiio);
-			io->scsiio.be_move_done(io);
+			ctl_datamove_done(io, true);
 			cfiscsi_session_terminate(cs);
 			return;
 		}
@@ -2622,7 +2622,7 @@ cfiscsi_datamove_in(union ctl_io *io)
 		cfiscsi_pdu_queue(response);
 	}
 
-	io->scsiio.be_move_done(io);
+	ctl_datamove_done(io, true);
 }
 
 static void
@@ -2651,9 +2651,10 @@ cfiscsi_datamove_out(union ctl_io *io)
 	 */
 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
 	if (io->scsiio.kern_rel_offset >= expected_len) {
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done(io, true);
 		return;
 	}
+
 	datamove_len = MIN(io->scsiio.kern_data_len,
 	    expected_len - io->scsiio.kern_rel_offset);
 
@@ -2669,7 +2670,7 @@ cfiscsi_datamove_out(union ctl_io *io)
 		CFISCSI_SESSION_WARN(cs, "failed to "
 		    "allocate memory; dropping connection");
 		ctl_set_busy(&io->scsiio);
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done(io, true);
 		cfiscsi_session_terminate(cs);
 		return;
 	}
@@ -2716,7 +2717,7 @@ cfiscsi_datamove_out(union ctl_io *io)
 		done = cfiscsi_handle_data_segment(request, cdw);
 		if (done) {
 			cfiscsi_data_wait_free(cs, cdw);
-			io->scsiio.be_move_done(io);
+			ctl_datamove_done(io, true);
 			return;
 		}
 	}
@@ -2739,7 +2740,7 @@ cfiscsi_datamove_out(union ctl_io *io)
 		CFISCSI_SESSION_WARN(cs, "failed to "
 		    "allocate memory; dropping connection");
 		ctl_set_busy(&io->scsiio);
-		io->scsiio.be_move_done(io);
+		ctl_datamove_done(io, true);
 		cfiscsi_session_terminate(cs);
 		return;
 	}
@@ -2920,7 +2921,7 @@ cfiscsi_task_management_done(union ctl_io *io)
 			    cdw, cdw_next);
 			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 			cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
-			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
+			ctl_datamove_done(cdw->cdw_ctl_io, false);
 			cfiscsi_data_wait_free(cs, cdw);
 		}
 		CFISCSI_SESSION_UNLOCK(cs);
diff --git a/sys/cam/ctl/ctl_io.h b/sys/cam/ctl/ctl_io.h
index eb63e5be1874..01887c20a822 100644
--- a/sys/cam/ctl/ctl_io.h
+++ b/sys/cam/ctl/ctl_io.h
@@ -328,7 +328,7 @@ struct ctl_scsiio {
 	ctl_tag_type tag_type;		/* simple, ordered, head of queue,etc.*/
 	uint8_t    cdb_len;		/* CDB length */
 	uint8_t	   cdb[CTL_MAX_CDBLEN];	/* CDB */
-	int	   (*be_move_done)(union ctl_io *io); /* called by fe */
+	int	   (*be_move_done)(union ctl_io *io, bool samethr); /* called by fe */
 	int        (*io_cont)(union ctl_io *io); /* to continue processing */
 };
 
diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c
index 60a8630218af..3a5e08f0a6ce 100644
--- a/sys/cam/ctl/ctl_tpc_local.c
+++ b/sys/cam/ctl/ctl_tpc_local.c
@@ -255,7 +255,7 @@ tpcl_datamove(union ctl_io *io)
 			 __func__, ctsio->ext_data_len, ctsio->kern_data_len));
 
 bailout:
-	io->scsiio.be_move_done(io);
+	ctl_datamove_done(io, true);
 }
 
 static void
diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c
index fc611fa641fa..dc0828e9911b 100644
--- a/sys/cam/ctl/scsi_ctl.c
+++ b/sys/cam/ctl/scsi_ctl.c
@@ -1396,8 +1396,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb)
 				xpt_release_ccb(done_ccb);
 				mtx_unlock(mtx);
 
-				/* Call the backend move done callback */
-				io->scsiio.be_move_done(io);
+				ctl_datamove_done(io, false);
 			}
 			return;
 		}
diff --git a/sys/dev/usb/storage/cfumass.c b/sys/dev/usb/storage/cfumass.c
index f424c977ce78..ff64440ac1eb 100644
--- a/sys/dev/usb/storage/cfumass.c
+++ b/sys/dev/usb/storage/cfumass.c
@@ -478,7 +478,7 @@ cfumass_terminate(struct cfumass_softc *sc)
 		if (sc->sc_ctl_io != NULL) {
 			CFUMASS_DEBUG(sc, "terminating CTL transfer");
 			ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio);
-			sc->sc_ctl_io->scsiio.be_move_done(sc->sc_ctl_io);
+			ctl_datamove_done(sc->sc_ctl_io, false);
 			sc->sc_ctl_io = NULL;
 		}
 
@@ -733,7 +733,7 @@ cfumass_t_data_callback(struct usb_xfer *xfer, usb_error_t usb_error)
 		    sc->sc_current_residue == 0 ||
 		    io->scsiio.kern_data_resid == 0) {
 			sc->sc_ctl_io = NULL;
-			io->scsiio.be_move_done(io);
+			ctl_datamove_done(io, false);
 			break;
 		}
 		/* FALLTHROUGH */
@@ -890,7 +890,7 @@ cfumass_datamove(union ctl_io *io)
 
 fail:
 	ctl_set_data_phase_error(&io->scsiio);
-	io->scsiio.be_move_done(io);
+	ctl_datamove_done(io, true);
 	sc->sc_ctl_io = NULL;
 }