busdma_bounce: Batch bounce page free operations when possible.

Reviewed by:	imp
Differential Revision:	https://reviews.freebsd.org/D34968

1 files changed, 34 insertions, 35 deletions

diff --git a/sys/kern/subr_busdma_bounce.c b/sys/kern/subr_busdma_bounce.c
index 243da8e9487f..f3699cf2ad27 100644
--- a/sys/kern/subr_busdma_bounce.c
+++ b/sys/kern/subr_busdma_bounce.c
@@ -382,38 +382,48 @@ add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 }
 
 static void
-free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
+free_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map)
 {
-	struct bus_dmamap *map;
+	struct bounce_page *bpage;
 	struct bounce_zone *bz;
 	bool schedule_thread;
+	u_int count;
+
+	if (STAILQ_EMPTY(&map->bpages))
+		return;
 
 	bz = dmat->bounce_zone;
-	bpage->datavaddr = 0;
-	bpage->datacount = 0;
-	if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) {
-		/*
-		 * Reset the bounce page to start at offset 0.  Other uses
-		 * of this bounce page may need to store a full page of
-		 * data and/or assume it starts on a page boundary.
-		 */
-		bpage->vaddr &= ~PAGE_MASK;
-		bpage->busaddr &= ~PAGE_MASK;
+	count = 0;
+	schedule_thread = false;
+	STAILQ_FOREACH(bpage, &map->bpages, links) {
+		bpage->datavaddr = 0;
+		bpage->datacount = 0;
+
+		if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) {
+			/*
+			 * Reset the bounce page to start at offset 0.
+			 * Other uses of this bounce page may need to
+			 * store a full page of data and/or assume it
+			 * starts on a page boundary.
+			 */
+			bpage->vaddr &= ~PAGE_MASK;
+			bpage->busaddr &= ~PAGE_MASK;
+		}
+		count++;
 	}
 
-	schedule_thread = false;
 	mtx_lock(&bounce_lock);
-	STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
-	bz->free_bpages++;
-	bz->active_bpages--;
-	if ((map = STAILQ_FIRST(&bz->bounce_map_waitinglist)) != NULL) {
-		if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
-			STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links);
-			STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
-			    map, links);
-			bz->total_deferred++;
-			schedule_thread = true;
-		}
+	STAILQ_CONCAT(&bz->bounce_page_list, &map->bpages);
+	bz->free_bpages += count;
+	bz->active_bpages -= count;
+	while ((map = STAILQ_FIRST(&bz->bounce_map_waitinglist)) != NULL) {
+		if (reserve_bounce_pages(map->dmat, map, 1) != 0)
+			break;
+
+		STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links);
+		STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links);
+		bz->total_deferred++;
+		schedule_thread = true;
 	}
 	mtx_unlock(&bounce_lock);
 	if (schedule_thread)
@@ -421,17 +431,6 @@ free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
 }
 
 static void
-free_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map)
-{
-	struct bounce_page *bpage;
-
-	while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
-		STAILQ_REMOVE_HEAD(&map->bpages, links);
-		free_bounce_page(dmat, bpage);
-	}
-}
-
-static void
 busdma_thread(void *dummy __unused)
 {
 	STAILQ_HEAD(, bus_dmamap) callbacklist;