Add support for the so-called streams feature of BULK endpoints

in SUPER-speed mode, USB 3.0.

This feature has not been tested yet, due to lack of hardware.

This feature is useful when implementing protocols like UASP,
USB attached SCSI which promises higher USB mass storage throughput.

This patch also implements support for hardware processing of endpoints
for increased performance. The switching to hardware processing
of an endpoint is done via a callback to the USB controller driver. The
stream feature is implemented like a variant of a hardware USB protocol.

USB controller drivers implementing device mode needs to be updated to
implement the new "xfer_stall" USB controller method and remove the
"xfer" argument from the "set_stall" method.

The API's toward existing USB drivers are preserved. To setup a USB transfer
in stream mode, set the "stream_id" field of the USB config structure to
the desired value.

The maximum number of BULK streams is currently hardcoded and limited to 8
via a define in usb_freebsd.h.

All USB drivers should be re-compiled after this change.

LibUSB will be updated next week to support streams mode. A new IOCTL to
setup BULK streams as already been implemented. The ugen device nodes
currently only supports stream ID zero.

The FreeBSD version has been bumped.

MFC after:	2 weeks

1 files changed, 94 insertions, 29 deletions

diff --git a/sys/dev/usb/controller/xhci.c b/sys/dev/usb/controller/xhci.c
index 1a4e43261b90..a20d7faacf13 100644
--- a/sys/dev/usb/controller/xhci.c
+++ b/sys/dev/usb/controller/xhci.c
@@ -128,7 +128,7 @@ static usb_proc_callback_t xhci_configure_msg;
 static usb_error_t xhci_configure_device(struct usb_device *);
 static usb_error_t xhci_configure_endpoint(struct usb_device *,
 		    struct usb_endpoint_descriptor *, uint64_t, uint16_t,
-		    uint8_t, uint8_t, uint8_t, uint16_t, uint16_t);
+		    uint8_t, uint8_t, uint8_t, uint16_t, uint16_t, uint8_t);
 static usb_error_t xhci_configure_mask(struct usb_device *,
 		    uint32_t, uint8_t);
 static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *,
@@ -1245,7 +1245,7 @@ xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
 		    &udev->ctrl_ep_desc);
 		err = xhci_configure_endpoint(udev,
 		    &udev->ctrl_ep_desc, pepext->physaddr,
-		    0, 1, 1, 0, mps, mps);
+		    0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT);
 
 		if (err != 0) {
 			DPRINTF("Could not configure default endpoint\n");
@@ -1800,7 +1800,8 @@ xhci_setup_generic_chain(struct usb_xfer *xfer)
 
 		/* compute multiplier for ISOCHRONOUS transfers */
 		mult = xfer->endpoint->ecomp ?
-		    (xfer->endpoint->ecomp->bmAttributes & 3) : 0;
+		    UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes)
+		    : 0;
 		/* check for USB 2.0 multiplier */
 		if (mult == 0) {
 			mult = (xfer->endpoint->edesc->
@@ -2055,7 +2056,8 @@ static usb_error_t
 xhci_configure_endpoint(struct usb_device *udev,
     struct usb_endpoint_descriptor *edesc, uint64_t ring_addr,
     uint16_t interval, uint8_t max_packet_count, uint8_t mult,
-    uint8_t fps_shift, uint16_t max_packet_size, uint16_t max_frame_size)
+    uint8_t fps_shift, uint16_t max_packet_size,
+    uint16_t max_frame_size, uint8_t ep_mode)
 {
 	struct usb_page_search buf_inp;
 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
@@ -2090,9 +2092,20 @@ xhci_configure_endpoint(struct usb_device *udev,
 	if (mult == 0)
 		return (USB_ERR_BAD_BUFSIZE);
 
-	temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
-	    XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
-	    XHCI_EPCTX_0_LSA_SET(0);
+	if (ep_mode == USB_EP_MODE_STREAMS) {
+		temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
+		    XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) |
+		    XHCI_EPCTX_0_LSA_SET(1);
+
+		ring_addr += sizeof(struct xhci_trb) *
+		    XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS;
+	} else {
+		temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
+		    XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
+		    XHCI_EPCTX_0_LSA_SET(0);
+
+		ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
+	}
 
 	switch (udev->speed) {
 	case USB_SPEED_FULL:
@@ -2160,9 +2173,6 @@ xhci_configure_endpoint(struct usb_device *udev,
 		temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
 
 	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp);
-
-	ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
-
 	xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr);
 
 	switch (edesc->bmAttributes & UE_XFERTYPE) {
@@ -2195,21 +2205,42 @@ xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer)
 {
 	struct xhci_endpoint_ext *pepext;
 	struct usb_endpoint_ss_comp_descriptor *ecomp;
+	usb_stream_t x;
 
 	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
 	    xfer->endpoint->edesc);
 
 	ecomp = xfer->endpoint->ecomp;
 
-	pepext->trb[0].dwTrb3 = 0;	/* halt any transfers */
+	for (x = 0; x != XHCI_MAX_STREAMS; x++) {
+		uint64_t temp;
+
+		/* halt any transfers */
+		pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0;
+
+		/* compute start of TRB ring for stream "x" */
+		temp = pepext->physaddr +
+		    (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) +
+		    XHCI_SCTX_0_SCT_SEC_TR_RING;
+
+		/* make tree structure */
+		pepext->trb[(XHCI_MAX_TRANSFERS *
+		    XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp);
+
+		/* reserved fields */
+		pepext->trb[(XHCI_MAX_TRANSFERS *
+                    XHCI_MAX_STREAMS) + x].dwTrb2 = 0;
+		pepext->trb[(XHCI_MAX_TRANSFERS *
+		    XHCI_MAX_STREAMS) + x].dwTrb3 = 0;
+	}
 	usb_pc_cpu_flush(pepext->page_cache);
 
 	return (xhci_configure_endpoint(xfer->xroot->udev,
 	    xfer->endpoint->edesc, pepext->physaddr,
 	    xfer->interval, xfer->max_packet_count,
-	    (ecomp != NULL) ? (ecomp->bmAttributes & 3) + 1 : 1,
+	    (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1,
 	    usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size,
-	    xfer->max_frame_size));
+	    xfer->max_frame_size, xfer->endpoint->ep_mode));
 }
 
 static usb_error_t
@@ -2500,7 +2531,8 @@ xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *e
 
 	pc = &sc->sc_hw.devs[index].endpoint_pc;
 
-	usbd_get_page(pc, (uintptr_t)&((struct xhci_dev_endpoint_trbs *)0)->trb[epno][0], &buf_ep);
+	usbd_get_page(pc, (uintptr_t)&((struct xhci_dev_endpoint_trbs *)0)->
+	    trb[epno][0], &buf_ep);
 
 	pepext = &sc->sc_hw.devs[index].endp[epno];
 	pepext->page_cache = pc;
@@ -2539,7 +2571,7 @@ xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error)
 		pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
 		    xfer->endpoint->edesc);
 
-		pepext->trb_used--;
+		pepext->trb_used[xfer->stream_id]--;
 
 		pepext->xfer[xfer->qh_pos] = NULL;
 
@@ -2557,12 +2589,15 @@ xhci_transfer_insert(struct usb_xfer *xfer)
 	struct xhci_td *td_last;
 	struct xhci_endpoint_ext *pepext;
 	uint64_t addr;
+	usb_stream_t id;
 	uint8_t i;
 	uint8_t inext;
 	uint8_t trb_limit;
 
 	DPRINTFN(8, "\n");
 
+	id = xfer->stream_id;
+
 	/* check if already inserted */
 	if (xfer->flags_int.bandwidth_reclaimed) {
 		DPRINTFN(8, "Already in schedule\n");
@@ -2588,7 +2623,7 @@ xhci_transfer_insert(struct usb_xfer *xfer)
 		break;
 	}
 
-	if (pepext->trb_used >= trb_limit) {
+	if (pepext->trb_used[id] >= trb_limit) {
 		DPRINTFN(8, "Too many TDs queued.\n");
 		return (USB_ERR_NOMEM);
 	}
@@ -2605,10 +2640,10 @@ xhci_transfer_insert(struct usb_xfer *xfer)
 		return (0);
 	}
 
-	pepext->trb_used++;
+	pepext->trb_used[id]++;
 
 	/* get current TRB index */
-	i = pepext->trb_index;
+	i = pepext->trb_index[id];
 
 	/* get next TRB index */
 	inext = (i + 1);
@@ -2617,8 +2652,12 @@ xhci_transfer_insert(struct usb_xfer *xfer)
 	if (inext >= (XHCI_MAX_TRANSFERS - 1))
 		inext = 0;
 
+	/* offset for stream */
+	i += id * XHCI_MAX_TRANSFERS;
+	inext += id * XHCI_MAX_TRANSFERS;
+
 	/* compute terminating return address */
-	addr += inext * sizeof(struct xhci_trb);
+	addr += (inext * sizeof(struct xhci_trb));
 
 	/* update next pointer of last link TRB */
 	td_last->td_trb[td_last->ntrb].qwTrb0 = htole64(addr);
@@ -2662,7 +2701,7 @@ xhci_transfer_insert(struct usb_xfer *xfer)
 
 	xfer->flags_int.bandwidth_reclaimed = 1;
 
-	pepext->trb_index = inext;
+	pepext->trb_index[id] = inext;
 
 	xhci_endpoint_doorbell(xfer);
 
@@ -2750,12 +2789,12 @@ xhci_device_generic_close(struct usb_xfer *xfer)
 
 static void
 xhci_device_generic_multi_enter(struct usb_endpoint *ep,
-    struct usb_xfer *enter_xfer)
+    usb_stream_t stream_id, struct usb_xfer *enter_xfer)
 {
 	struct usb_xfer *xfer;
 
 	/* check if there is a current transfer */
-	xfer = ep->endpoint_q.curr;
+	xfer = ep->endpoint_q[stream_id].curr;
 	if (xfer == NULL)
 		return;
 
@@ -2767,7 +2806,7 @@ xhci_device_generic_multi_enter(struct usb_endpoint *ep,
 	if (!xfer->flags_int.bandwidth_reclaimed)
 		return;
 
-	xfer = TAILQ_FIRST(&ep->endpoint_q.head);
+	xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head);
 	if (xfer == NULL) {
 		/*
 		 * In case of enter we have to consider that the
@@ -2792,7 +2831,8 @@ xhci_device_generic_enter(struct usb_xfer *xfer)
 	/* setup TD's and QH */
 	xhci_setup_generic_chain(xfer);
 
-	xhci_device_generic_multi_enter(xfer->endpoint, xfer);
+	xhci_device_generic_multi_enter(xfer->endpoint,
+	    xfer->stream_id, xfer);
 }
 
 static void
@@ -2804,7 +2844,8 @@ xhci_device_generic_start(struct usb_xfer *xfer)
 	xhci_transfer_insert(xfer);
 
 	/* try to multi buffer */
-	xhci_device_generic_multi_enter(xfer->endpoint, NULL);
+	xhci_device_generic_multi_enter(xfer->endpoint,
+	    xfer->stream_id, NULL);
 
 	/* add transfer last on interrupt queue */
 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
@@ -3465,6 +3506,7 @@ xhci_configure_reset_endpoint(struct usb_xfer *xfer)
 	struct usb_endpoint_descriptor *edesc;
 	struct usb_page_cache *pcinp;
 	usb_error_t err;
+	usb_stream_t stream_id;
 	uint8_t index;
 	uint8_t epno;
 
@@ -3481,6 +3523,7 @@ xhci_configure_reset_endpoint(struct usb_xfer *xfer)
 	edesc = xfer->endpoint->edesc;
 
 	epno = edesc->bEndpointAddress;
+	stream_id = xfer->stream_id;
 
 	if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
 		epno |= UE_DIR_IN;
@@ -3516,8 +3559,10 @@ xhci_configure_reset_endpoint(struct usb_xfer *xfer)
 	if (err != 0)
 		DPRINTF("Could not reset endpoint %u\n", epno);
 
-	err = xhci_cmd_set_tr_dequeue_ptr(sc, pepext->physaddr |
-	    XHCI_EPCTX_2_DCS_SET(1), 0, epno, index);
+	err = xhci_cmd_set_tr_dequeue_ptr(sc,
+	    (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) *
+	    XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1),
+	    stream_id, epno, index);
 
 	if (err != 0)
 		DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno);
@@ -3615,7 +3660,8 @@ restart:
 			/* check if halted is still cleared */
 			if (pepext->trb_halted == 0) {
 				pepext->trb_running = 1;
-				pepext->trb_index = 0;
+				memset(pepext->trb_index, 0,
+				    sizeof(pepext->trb_index));
 			}
 			goto restart;
 		}
@@ -3639,7 +3685,8 @@ restart:
 		xhci_transfer_insert(xfer);
 
 		/* try to multi buffer */
-		xhci_device_generic_multi_enter(xfer->endpoint, NULL);
+		xhci_device_generic_multi_enter(xfer->endpoint,
+		    xfer->stream_id, NULL);
 	}
 }
 
@@ -3946,6 +3993,23 @@ xhci_device_state_change(struct usb_device *udev)
 	XHCI_CMD_UNLOCK(sc);
 }
 
+static usb_error_t
+xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep,
+    uint8_t ep_mode)
+{
+	switch (ep_mode) {
+	case USB_EP_MODE_DEFAULT:
+		return (0);
+	case USB_EP_MODE_STREAMS:
+		if ((ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK ||
+		    udev->speed != USB_SPEED_SUPER)
+			return (USB_ERR_INVAL);
+		return (0);
+	default:
+		return (USB_ERR_INVAL);
+	}
+}
+
 struct usb_bus_methods xhci_bus_methods = {
 	.endpoint_init = xhci_ep_init,
 	.endpoint_uninit = xhci_ep_uninit,
@@ -3964,4 +4028,5 @@ struct usb_bus_methods xhci_bus_methods = {
 	.clear_stall = xhci_ep_clear_stall,
 	.device_state_change = xhci_device_state_change,
 	.set_hw_power_sleep = xhci_set_hw_power_sleep,
+	.set_endpoint_mode = xhci_set_endpoint_mode,
 };