diff options
author | Andrew Thompson <thompsa@FreeBSD.org> | 2009-02-23 18:31:00 +0000 |
---|---|---|
committer | Andrew Thompson <thompsa@FreeBSD.org> | 2009-02-23 18:31:00 +0000 |
commit | 02ac6454880b59bbc5f3f74dffaffa90b30adc8b (patch) | |
tree | 691e9b9009214e6138d3913e4c022c897e386667 /sys/dev/usb/usb_busdma.c | |
parent | ab8b490a63599c4b0a045c989c1a7162b161d2a5 (diff) | |
download | src-02ac6454880b59bbc5f3f74dffaffa90b30adc8b.tar.gz src-02ac6454880b59bbc5f3f74dffaffa90b30adc8b.zip |
Move the new USB stack into its new home.
Notes
Notes:
svn path=/head/; revision=188942
Diffstat (limited to 'sys/dev/usb/usb_busdma.c')
-rw-r--r-- | sys/dev/usb/usb_busdma.c | 1426 |
1 files changed, 1426 insertions, 0 deletions
diff --git a/sys/dev/usb/usb_busdma.c b/sys/dev/usb/usb_busdma.c new file mode 100644 index 000000000000..809c3bfea0a3 --- /dev/null +++ b/sys/dev/usb/usb_busdma.c @@ -0,0 +1,1426 @@ +/* $FreeBSD$ */ +/*- + * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <dev/usb/usb_mfunc.h> +#include <dev/usb/usb_error.h> +#include <dev/usb/usb.h> +#include <dev/usb/usb_defs.h> + +#define USB_DEBUG_VAR usb2_debug + +#include <dev/usb/usb_core.h> +#include <dev/usb/usb_busdma.h> +#include <dev/usb/usb_process.h> +#include <dev/usb/usb_transfer.h> +#include <dev/usb/usb_device.h> +#include <dev/usb/usb_util.h> +#include <dev/usb/usb_debug.h> + +#include <dev/usb/usb_controller.h> +#include <dev/usb/usb_bus.h> + +static void usb2_dma_tag_create(struct usb2_dma_tag *, uint32_t, uint32_t); +static void usb2_dma_tag_destroy(struct usb2_dma_tag *); + +#ifdef __FreeBSD__ +static void usb2_dma_lock_cb(void *, bus_dma_lock_op_t); +static int32_t usb2_m_copy_in_cb(void *, void *, uint32_t); +static void usb2_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); +static void usb2_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); +static void usb2_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, + uint8_t); +#endif + +#ifdef __NetBSD__ +static int32_t usb2_m_copy_in_cb(void *, caddr_t, uint32_t); +static void usb2_pc_common_mem_cb(struct usb2_page_cache *, + bus_dma_segment_t *, int, int, uint8_t); +#endif + +/*------------------------------------------------------------------------* + * usb2_get_page - lookup DMA-able memory for the given offset + * + * NOTE: Only call this function when the "page_cache" structure has + * been properly initialized ! + *------------------------------------------------------------------------*/ +void +usb2_get_page(struct usb2_page_cache *pc, uint32_t offset, + struct usb2_page_search *res) +{ + struct usb2_page *page; + + if (pc->page_start) { + + /* Case 1 - something has been loaded into DMA */ + + if (pc->buffer) { + + /* Case 1a - Kernel Virtual Address */ + + res->buffer = USB_ADD_BYTES(pc->buffer, offset); + } + offset += pc->page_offset_buf; + + /* compute destination page */ + + page = pc->page_start; + + if (pc->ismultiseg) { + + page += (offset / USB_PAGE_SIZE); + + offset %= USB_PAGE_SIZE; + + res->length = USB_PAGE_SIZE - offset; + res->physaddr = page->physaddr + offset; + } else { + res->length = 0 - 1; + res->physaddr = page->physaddr + offset; + } + if (!pc->buffer) { + + /* Case 1b - Non Kernel Virtual Address */ + + res->buffer = USB_ADD_BYTES(page->buffer, offset); + } + } else { + + /* Case 2 - Plain PIO */ + + res->buffer = USB_ADD_BYTES(pc->buffer, offset); + res->length = 0 - 1; + res->physaddr = 0; + } +} + +/*------------------------------------------------------------------------* + * usb2_copy_in - copy directly to DMA-able memory + *------------------------------------------------------------------------*/ +void +usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset, + const void *ptr, uint32_t len) +{ + struct usb2_page_search buf_res; + + while (len != 0) { + + usb2_get_page(cache, offset, &buf_res); + + if (buf_res.length > len) { + buf_res.length = len; + } + bcopy(ptr, buf_res.buffer, buf_res.length); + + offset += buf_res.length; + len -= buf_res.length; + ptr = USB_ADD_BYTES(ptr, buf_res.length); + } +} + +/*------------------------------------------------------------------------* + * usb2_copy_in_user - copy directly to DMA-able memory from userland + * + * Return values: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +int +usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset, + const void *ptr, uint32_t len) +{ + struct usb2_page_search buf_res; + int error; + + while (len != 0) { + + usb2_get_page(cache, offset, &buf_res); + + if (buf_res.length > len) { + buf_res.length = len; + } + error = copyin(ptr, buf_res.buffer, buf_res.length); + if (error) + return (error); + + offset += buf_res.length; + len -= buf_res.length; + ptr = USB_ADD_BYTES(ptr, buf_res.length); + } + return (0); /* success */ +} + +/*------------------------------------------------------------------------* + * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory + *------------------------------------------------------------------------*/ +struct usb2_m_copy_in_arg { + struct usb2_page_cache *cache; + uint32_t dst_offset; +}; + +static int32_t +#ifdef __FreeBSD__ +usb2_m_copy_in_cb(void *arg, void *src, uint32_t count) +#else +usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count) +#endif +{ + register struct usb2_m_copy_in_arg *ua = arg; + + usb2_copy_in(ua->cache, ua->dst_offset, src, count); + ua->dst_offset += count; + return (0); +} + +void +usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset, + struct mbuf *m, uint32_t src_offset, uint32_t src_len) +{ + struct usb2_m_copy_in_arg arg = {cache, dst_offset}; + register int error; + + error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg); +} + +/*------------------------------------------------------------------------* + * usb2_uiomove - factored out code + *------------------------------------------------------------------------*/ +int +usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio, + uint32_t pc_offset, uint32_t len) +{ + struct usb2_page_search res; + int error = 0; + + while (len != 0) { + + usb2_get_page(pc, pc_offset, &res); + + if (res.length > len) { + res.length = len; + } + /* + * "uiomove()" can sleep so one needs to make a wrapper, + * exiting the mutex and checking things + */ + error = uiomove(res.buffer, res.length, uio); + + if (error) { + break; + } + pc_offset += res.length; + len -= res.length; + } + return (error); +} + +/*------------------------------------------------------------------------* + * usb2_copy_out - copy directly from DMA-able memory + *------------------------------------------------------------------------*/ +void +usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset, + void *ptr, uint32_t len) +{ + struct usb2_page_search res; + + while (len != 0) { + + usb2_get_page(cache, offset, &res); + + if (res.length > len) { + res.length = len; + } + bcopy(res.buffer, ptr, res.length); + + offset += res.length; + len -= res.length; + ptr = USB_ADD_BYTES(ptr, res.length); + } +} + +/*------------------------------------------------------------------------* + * usb2_copy_out_user - copy directly from DMA-able memory to userland + * + * Return values: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +int +usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset, + void *ptr, uint32_t len) +{ + struct usb2_page_search res; + int error; + + while (len != 0) { + + usb2_get_page(cache, offset, &res); + + if (res.length > len) { + res.length = len; + } + error = copyout(res.buffer, ptr, res.length); + if (error) + return (error); + + offset += res.length; + len -= res.length; + ptr = USB_ADD_BYTES(ptr, res.length); + } + return (0); /* success */ +} + +/*------------------------------------------------------------------------* + * usb2_bzero - zero DMA-able memory + *------------------------------------------------------------------------*/ +void +usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len) +{ + struct usb2_page_search res; + + while (len != 0) { + + usb2_get_page(cache, offset, &res); + + if (res.length > len) { + res.length = len; + } + bzero(res.buffer, res.length); + + offset += res.length; + len -= res.length; + } +} + + +#ifdef __FreeBSD__ + +/*------------------------------------------------------------------------* + * usb2_dma_lock_cb - dummy callback + *------------------------------------------------------------------------*/ +static void +usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op) +{ + /* we use "mtx_owned()" instead of this function */ +} + +/*------------------------------------------------------------------------* + * usb2_dma_tag_create - allocate a DMA tag + * + * NOTE: If the "align" parameter has a value of 1 the DMA-tag will + * allow multi-segment mappings. Else all mappings are single-segment. + *------------------------------------------------------------------------*/ +static void +usb2_dma_tag_create(struct usb2_dma_tag *udt, + uint32_t size, uint32_t align) +{ + bus_dma_tag_t tag; + + if (bus_dma_tag_create + ( /* parent */ udt->tag_parent->tag, + /* alignment */ align, + /* boundary */ USB_PAGE_SIZE, + /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, + /* highaddr */ BUS_SPACE_MAXADDR, + /* filter */ NULL, + /* filterarg */ NULL, + /* maxsize */ size, + /* nsegments */ (align == 1) ? + (2 + (size / USB_PAGE_SIZE)) : 1, + /* maxsegsz */ (align == 1) ? + USB_PAGE_SIZE : size, + /* flags */ BUS_DMA_KEEP_PG_OFFSET, + /* lockfn */ &usb2_dma_lock_cb, + /* lockarg */ NULL, + &tag)) { + tag = NULL; + } + udt->tag = tag; +} + +/*------------------------------------------------------------------------* + * usb2_dma_tag_free - free a DMA tag + *------------------------------------------------------------------------*/ +static void +usb2_dma_tag_destroy(struct usb2_dma_tag *udt) +{ + bus_dma_tag_destroy(udt->tag); +} + +/*------------------------------------------------------------------------* + * usb2_pc_alloc_mem_cb - BUS-DMA callback function + *------------------------------------------------------------------------*/ +static void +usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, + int nseg, int error) +{ + usb2_pc_common_mem_cb(arg, segs, nseg, error, 0); +} + +/*------------------------------------------------------------------------* + * usb2_pc_load_mem_cb - BUS-DMA callback function + *------------------------------------------------------------------------*/ +static void +usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, + int nseg, int error) +{ + usb2_pc_common_mem_cb(arg, segs, nseg, error, 1); +} + +/*------------------------------------------------------------------------* + * usb2_pc_common_mem_cb - BUS-DMA callback function + *------------------------------------------------------------------------*/ +static void +usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, + int nseg, int error, uint8_t isload) +{ + struct usb2_dma_parent_tag *uptag; + struct usb2_page_cache *pc; + struct usb2_page *pg; + uint32_t rem; + uint8_t owned; + + pc = arg; + uptag = pc->tag_parent; + + /* + * XXX There is sometimes recursive locking here. + * XXX We should try to find a better solution. + * XXX Until further the "owned" variable does + * XXX the trick. + */ + + if (error) { + goto done; + } + pg = pc->page_start; + pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); + rem = segs->ds_addr & (USB_PAGE_SIZE - 1); + pc->page_offset_buf = rem; + pc->page_offset_end += rem; + nseg--; +#if (USB_DEBUG != 0) + if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { + /* + * This check verifies that the physical address is correct: + */ + DPRINTFN(0, "Page offset was not preserved!\n"); + error = 1; + goto done; + } +#endif + while (nseg > 0) { + nseg--; + segs++; + pg++; + pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); + } + +done: + owned = mtx_owned(uptag->mtx); + if (!owned) + mtx_lock(uptag->mtx); + + uptag->dma_error = (error ? 1 : 0); + if (isload) { + (uptag->func) (uptag); + } else { + usb2_cv_broadcast(uptag->cv); + } + if (!owned) + mtx_unlock(uptag->mtx); +} + +/*------------------------------------------------------------------------* + * usb2_pc_alloc_mem - allocate DMA'able memory + * + * Returns: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +uint8_t +usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, + uint32_t size, uint32_t align) +{ + struct usb2_dma_parent_tag *uptag; + struct usb2_dma_tag *utag; + bus_dmamap_t map; + void *ptr; + int err; + + uptag = pc->tag_parent; + + if (align != 1) { + /* + * The alignment must be greater or equal to the + * "size" else the object can be split between two + * memory pages and we get a problem! + */ + while (align < size) { + align *= 2; + if (align == 0) { + goto error; + } + } +#if 1 + /* + * XXX BUS-DMA workaround - FIXME later: + * + * We assume that that the aligment at this point of + * the code is greater than or equal to the size and + * less than two times the size, so that if we double + * the size, the size will be greater than the + * alignment. + * + * The bus-dma system has a check for "alignment" + * being less than "size". If that check fails we end + * up using contigmalloc which is page based even for + * small allocations. Try to avoid that to save + * memory, hence we sometimes to a large number of + * small allocations! + */ + if (size <= (USB_PAGE_SIZE / 2)) { + size *= 2; + } +#endif + } + /* get the correct DMA tag */ + utag = usb2_dma_tag_find(uptag, size, align); + if (utag == NULL) { + goto error; + } + /* allocate memory */ + if (bus_dmamem_alloc( + utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { + goto error; + } + /* setup page cache */ + pc->buffer = ptr; + pc->page_start = pg; + pc->page_offset_buf = 0; + pc->page_offset_end = size; + pc->map = map; + pc->tag = utag->tag; + pc->ismultiseg = (align == 1); + + mtx_lock(uptag->mtx); + + /* load memory into DMA */ + err = bus_dmamap_load( + utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb, + pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); + + if (err == EINPROGRESS) { + usb2_cv_wait(uptag->cv, uptag->mtx); + err = 0; + } + mtx_unlock(uptag->mtx); + + if (err || uptag->dma_error) { + bus_dmamem_free(utag->tag, ptr, map); + goto error; + } + bzero(ptr, size); + + usb2_pc_cpu_flush(pc); + + return (0); + +error: + /* reset most of the page cache */ + pc->buffer = NULL; + pc->page_start = NULL; + pc->page_offset_buf = 0; + pc->page_offset_end = 0; + pc->map = NULL; + pc->tag = NULL; + return (1); +} + +/*------------------------------------------------------------------------* + * usb2_pc_free_mem - free DMA memory + * + * This function is NULL safe. + *------------------------------------------------------------------------*/ +void +usb2_pc_free_mem(struct usb2_page_cache *pc) +{ + if (pc && pc->buffer) { + + bus_dmamap_unload(pc->tag, pc->map); + + bus_dmamem_free(pc->tag, pc->buffer, pc->map); + + pc->buffer = NULL; + } +} + +/*------------------------------------------------------------------------* + * usb2_pc_load_mem - load virtual memory into DMA + * + * Return values: + * 0: Success + * Else: Error + *------------------------------------------------------------------------*/ +uint8_t +usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync) +{ + /* setup page cache */ + pc->page_offset_buf = 0; + pc->page_offset_end = size; + pc->ismultiseg = 1; + + mtx_assert(pc->tag_parent->mtx, MA_OWNED); + + if (size > 0) { + if (sync) { + struct usb2_dma_parent_tag *uptag; + int err; + + uptag = pc->tag_parent; + + /* + * We have to unload the previous loaded DMA + * pages before trying to load a new one! + */ + bus_dmamap_unload(pc->tag, pc->map); + + /* + * Try to load memory into DMA. + */ + err = bus_dmamap_load( + pc->tag, pc->map, pc->buffer, size, + &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); + if (err == EINPROGRESS) { + usb2_cv_wait(uptag->cv, uptag->mtx); + err = 0; + } + if (err || uptag->dma_error) { + return (1); + } + } else { + + /* + * We have to unload the previous loaded DMA + * pages before trying to load a new one! + */ + bus_dmamap_unload(pc->tag, pc->map); + + /* + * Try to load memory into DMA. The callback + * will be called in all cases: + */ + if (bus_dmamap_load( + pc->tag, pc->map, pc->buffer, size, + &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { + } + } + } else { + if (!sync) { + /* + * Call callback so that refcount is decremented + * properly: + */ + pc->tag_parent->dma_error = 0; + (pc->tag_parent->func) (pc->tag_parent); + } + } + return (0); +} + +/*------------------------------------------------------------------------* + * usb2_pc_cpu_invalidate - invalidate CPU cache + *------------------------------------------------------------------------*/ +void +usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) +{ + if (pc->page_offset_end == pc->page_offset_buf) { + /* nothing has been loaded into this page cache! */ + return; + } + bus_dmamap_sync(pc->tag, pc->map, + BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); +} + +/*------------------------------------------------------------------------* + * usb2_pc_cpu_flush - flush CPU cache + *------------------------------------------------------------------------*/ +void +usb2_pc_cpu_flush(struct usb2_page_cache *pc) +{ + if (pc->page_offset_end == pc->page_offset_buf) { + /* nothing has been loaded into this page cache! */ + return; + } + bus_dmamap_sync(pc->tag, pc->map, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); +} + +/*------------------------------------------------------------------------* + * usb2_pc_dmamap_create - create a DMA map + * + * Returns: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +uint8_t +usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size) +{ + struct usb2_xfer_root *info; + struct usb2_dma_tag *utag; + + /* get info */ + info = pc->tag_parent->info; + + /* sanity check */ + if (info == NULL) { + goto error; + } + utag = usb2_dma_tag_find(pc->tag_parent, size, 1); + if (utag == NULL) { + goto error; + } + /* create DMA map */ + if (bus_dmamap_create(utag->tag, 0, &pc->map)) { + goto error; + } + pc->tag = utag->tag; + return 0; /* success */ + +error: + pc->map = NULL; + pc->tag = NULL; + return 1; /* failure */ +} + +/*------------------------------------------------------------------------* + * usb2_pc_dmamap_destroy + * + * This function is NULL safe. + *------------------------------------------------------------------------*/ +void +usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) +{ + if (pc && pc->tag) { + bus_dmamap_destroy(pc->tag, pc->map); + pc->tag = NULL; + pc->map = NULL; + } +} + +#endif + +#ifdef __NetBSD__ + +/*------------------------------------------------------------------------* + * usb2_dma_tag_create - allocate a DMA tag + * + * NOTE: If the "align" parameter has a value of 1 the DMA-tag will + * allow multi-segment mappings. Else all mappings are single-segment. + *------------------------------------------------------------------------*/ +static void +usb2_dma_tag_create(struct usb2_dma_tag *udt, + uint32_t size, uint32_t align) +{ + uint32_t nseg; + + if (align == 1) { + nseg = (2 + (size / USB_PAGE_SIZE)); + } else { + nseg = 1; + } + + udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)), + M_USB, M_WAITOK | M_ZERO); + + if (udt->p_seg == NULL) { + return; + } + udt->tag = udt->tag_parent->tag; + udt->n_seg = nseg; +} + +/*------------------------------------------------------------------------* + * usb2_dma_tag_free - free a DMA tag + *------------------------------------------------------------------------*/ +static void +usb2_dma_tag_destroy(struct usb2_dma_tag *udt) +{ + free(udt->p_seg, M_USB); +} + +/*------------------------------------------------------------------------* + * usb2_pc_common_mem_cb - BUS-DMA callback function + *------------------------------------------------------------------------*/ +static void +usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, + int nseg, int error, uint8_t isload, uint8_t dolock) +{ + struct usb2_dma_parent_tag *uptag; + struct usb2_page *pg; + uint32_t rem; + uint8_t ext_seg; /* extend last segment */ + + uptag = pc->tag_parent; + + if (error) { + goto done; + } + pg = pc->page_start; + pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); + rem = segs->ds_addr & (USB_PAGE_SIZE - 1); + pc->page_offset_buf = rem; + pc->page_offset_end += rem; + if (nseg < ((pc->page_offset_end + + (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) { + ext_seg = 1; + } else { + ext_seg = 0; + } + nseg--; +#if (USB_DEBUG != 0) + if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { + /* + * This check verifies that the physical address is correct: + */ + DPRINTFN(0, "Page offset was not preserved!\n"); + error = 1; + goto done; + } +#endif + while (nseg > 0) { + nseg--; + segs++; + pg++; + pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); + } + + /* + * XXX The segments we get from BUS-DMA are not aligned, + * XXX so we need to extend the last segment if we are + * XXX unaligned and cross the segment boundary! + */ + if (ext_seg && pc->ismultiseg) { + (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE; + } +done: + if (dolock) + mtx_lock(uptag->mtx); + + uptag->dma_error = (error ? 1 : 0); + if (isload) { + (uptag->func) (uptag); + } + if (dolock) + mtx_unlock(uptag->mtx); +} + +/*------------------------------------------------------------------------* + * usb2_pc_alloc_mem - allocate DMA'able memory + * + * Returns: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +uint8_t +usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, + uint32_t size, uint32_t align) +{ + struct usb2_dma_parent_tag *uptag; + struct usb2_dma_tag *utag; + caddr_t ptr = NULL; + bus_dmamap_t map; + int seg_count; + + uptag = pc->tag_parent; + + if (align != 1) { + /* + * The alignment must be greater or equal to the + * "size" else the object can be split between two + * memory pages and we get a problem! + */ + while (align < size) { + align *= 2; + if (align == 0) { + goto done_5; + } + } + } + /* get the correct DMA tag */ + utag = usb2_dma_tag_find(pc->tag_parent, size, align); + if (utag == NULL) { + goto done_5; + } + if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg, + utag->n_seg, &seg_count, BUS_DMA_WAITOK)) { + goto done_4; + } + if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size, + &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) { + goto done_3; + } + if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ? + USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) { + goto done_2; + } + if (bus_dmamap_load(utag->tag, map, ptr, size, NULL, + BUS_DMA_WAITOK)) { + goto done_1; + } + pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)), + M_USB, M_WAITOK | M_ZERO); + if (pc->p_seg == NULL) { + goto done_0; + } + /* store number if actual segments used */ + pc->n_seg = seg_count; + + /* make a copy of the segments */ + bcopy(utag->p_seg, pc->p_seg, + seg_count * sizeof(*(pc->p_seg))); + + /* setup page cache */ + pc->buffer = ptr; + pc->page_start = pg; + pc->page_offset_buf = 0; + pc->page_offset_end = size; + pc->map = map; + pc->tag = utag->tag; + pc->ismultiseg = (align == 1); + + usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1); + + bzero(ptr, size); + + usb2_pc_cpu_flush(pc); + + return (0); + +done_0: + bus_dmamap_unload(utag->tag, map); +done_1: + bus_dmamap_destroy(utag->tag, map); +done_2: + bus_dmamem_unmap(utag->tag, ptr, size); +done_3: + bus_dmamem_free(utag->tag, utag->p_seg, seg_count); +done_4: + /* utag is destroyed later */ +done_5: + /* reset most of the page cache */ + pc->buffer = NULL; + pc->page_start = NULL; + pc->page_offset_buf = 0; + pc->page_offset_end = 0; + pc->map = NULL; + pc->tag = NULL; + pc->n_seg = 0; + pc->p_seg = NULL; + return (1); +} + +/*------------------------------------------------------------------------* + * usb2_pc_free_mem - free DMA memory + * + * This function is NULL safe. + *------------------------------------------------------------------------*/ +void +usb2_pc_free_mem(struct usb2_page_cache *pc) +{ + if (pc && pc->buffer) { + bus_dmamap_unload(pc->tag, pc->map); + bus_dmamap_destroy(pc->tag, pc->map); + bus_dmamem_unmap(pc->tag, pc->buffer, + pc->page_offset_end - pc->page_offset_buf); + bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg); + free(pc->p_seg, M_USB); + pc->buffer = NULL; + } +} + +/*------------------------------------------------------------------------* + * usb2_pc_load_mem - load virtual memory into DMA + * + * Return values: + * 0: Success + * Else: Error + *------------------------------------------------------------------------*/ +uint8_t +usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync) +{ + int error; + + /* setup page cache */ + pc->page_offset_buf = 0; + pc->page_offset_end = size; + pc->ismultiseg = 1; + + if (size > 0) { + + /* + * We have to unload the previous loaded DMA + * pages before trying to load a new one! + */ + bus_dmamap_unload(pc->tag, pc->map); + + /* try to load memory into DMA using using no wait option */ + if (bus_dmamap_load(pc->tag, pc->map, pc->buffer, + size, NULL, BUS_DMA_NOWAIT)) { + error = ENOMEM; + } else { + error = 0; + } + + usb2_pc_common_mem_cb(pc, pc->map->dm_segs, + pc->map->dm_nsegs, error, !sync); + + if (error) { + return (1); + } + } else { + if (!sync) { + /* + * Call callback so that refcount is decremented + * properly: + */ + pc->tag_parent->dma_error = 0; + (pc->tag_parent->func) (pc->tag_parent); + } + } + return (0); +} + +/*------------------------------------------------------------------------* + * usb2_pc_cpu_invalidate - invalidate CPU cache + *------------------------------------------------------------------------*/ +void +usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) +{ + uint32_t len; + + len = pc->page_offset_end - pc->page_offset_buf; + + if (len == 0) { + /* nothing has been loaded into this page cache */ + return; + } + bus_dmamap_sync(pc->tag, pc->map, 0, len, + BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); +} + +/*------------------------------------------------------------------------* + * usb2_pc_cpu_flush - flush CPU cache + *------------------------------------------------------------------------*/ +void +usb2_pc_cpu_flush(struct usb2_page_cache *pc) +{ + uint32_t len; + + len = pc->page_offset_end - pc->page_offset_buf; + + if (len == 0) { + /* nothing has been loaded into this page cache */ + return; + } + bus_dmamap_sync(pc->tag, pc->map, 0, len, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); +} + +/*------------------------------------------------------------------------* + * usb2_pc_dmamap_create - create a DMA map + * + * Returns: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +uint8_t +usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size) +{ + struct usb2_xfer_root *info; + struct usb2_dma_tag *utag; + + /* get info */ + info = pc->tag_parent->info; + + /* sanity check */ + if (info == NULL) { + goto error; + } + utag = usb2_dma_tag_find(pc->tag_parent, size, 1); + if (utag == NULL) { + goto error; + } + if (bus_dmamap_create(utag->tag, size, utag->n_seg, + USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) { + goto error; + } + pc->tag = utag->tag; + pc->p_seg = utag->p_seg; + pc->n_seg = utag->n_seg; + return 0; /* success */ + +error: + pc->map = NULL; + pc->tag = NULL; + pc->p_seg = NULL; + pc->n_seg = 0; + return 1; /* failure */ +} + +/*------------------------------------------------------------------------* + * usb2_pc_dmamap_destroy + * + * This function is NULL safe. + *------------------------------------------------------------------------*/ +void +usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) +{ + if (pc && pc->tag) { + bus_dmamap_destroy(pc->tag, pc->map); + pc->tag = NULL; + pc->map = NULL; + } +} + +#endif + +/*------------------------------------------------------------------------* + * usb2_dma_tag_find - factored out code + *------------------------------------------------------------------------*/ +struct usb2_dma_tag * +usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt, + uint32_t size, uint32_t align) +{ + struct usb2_dma_tag *udt; + uint8_t nudt; + + USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n")); + USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n")); + + udt = udpt->utag_first; + nudt = udpt->utag_max; + + while (nudt--) { + + if (udt->align == 0) { + usb2_dma_tag_create(udt, size, align); + if (udt->tag == NULL) { + return (NULL); + } + udt->align = align; + udt->size = size; + return (udt); + } + if ((udt->align == align) && (udt->size == size)) { + return (udt); + } + udt++; + } + return (NULL); +} + +/*------------------------------------------------------------------------* + * usb2_dma_tag_setup - initialise USB DMA tags + *------------------------------------------------------------------------*/ +void +usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt, + struct usb2_dma_tag *udt, bus_dma_tag_t dmat, + struct mtx *mtx, usb2_dma_callback_t *func, + struct usb2_xfer_root *info, uint8_t ndmabits, + uint8_t nudt) +{ + bzero(udpt, sizeof(*udpt)); + + /* sanity checking */ + if ((nudt == 0) || + (ndmabits == 0) || + (mtx == NULL)) { + /* something is corrupt */ + return; + } +#ifdef __FreeBSD__ + /* initialise condition variable */ + usb2_cv_init(udpt->cv, "USB DMA CV"); +#endif + + /* store some information */ + udpt->mtx = mtx; + udpt->info = info; + udpt->func = func; + udpt->tag = dmat; + udpt->utag_first = udt; + udpt->utag_max = nudt; + udpt->dma_bits = ndmabits; + + while (nudt--) { + bzero(udt, sizeof(*udt)); + udt->tag_parent = udpt; + udt++; + } +} + +/*------------------------------------------------------------------------* + * usb2_bus_tag_unsetup - factored out code + *------------------------------------------------------------------------*/ +void +usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt) +{ + struct usb2_dma_tag *udt; + uint8_t nudt; + + udt = udpt->utag_first; + nudt = udpt->utag_max; + + while (nudt--) { + + if (udt->align) { + /* destroy the USB DMA tag */ + usb2_dma_tag_destroy(udt); + udt->align = 0; + } + udt++; + } + + if (udpt->utag_max) { +#ifdef __FreeBSD__ + /* destroy the condition variable */ + usb2_cv_destroy(udpt->cv); +#endif + } +} + +/*------------------------------------------------------------------------* + * usb2_bdma_work_loop + * + * This function handles loading of virtual buffers into DMA and is + * only called when "dma_refcount" is zero. + *------------------------------------------------------------------------*/ +void +usb2_bdma_work_loop(struct usb2_xfer_queue *pq) +{ + struct usb2_xfer_root *info; + struct usb2_xfer *xfer; + uint32_t nframes; + + xfer = pq->curr; + info = xfer->xroot; + + mtx_assert(info->xfer_mtx, MA_OWNED); + + if (xfer->error) { + /* some error happened */ + USB_BUS_LOCK(info->bus); + usb2_transfer_done(xfer, 0); + USB_BUS_UNLOCK(info->bus); + return; + } + if (!xfer->flags_int.bdma_setup) { + struct usb2_page *pg; + uint32_t frlength_0; + uint8_t isread; + + xfer->flags_int.bdma_setup = 1; + + /* reset BUS-DMA load state */ + + info->dma_error = 0; + + if (xfer->flags_int.isochronous_xfr) { + /* only one frame buffer */ + nframes = 1; + frlength_0 = xfer->sumlen; + } else { + /* can be multiple frame buffers */ + nframes = xfer->nframes; + frlength_0 = xfer->frlengths[0]; + } + + /* + * Set DMA direction first. This is needed to + * select the correct cache invalidate and cache + * flush operations. + */ + isread = USB_GET_DATA_ISREAD(xfer); + pg = xfer->dma_page_ptr; + + if (xfer->flags_int.control_xfr && + xfer->flags_int.control_hdr) { + /* special case */ + if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) { + /* The device controller writes to memory */ + xfer->frbuffers[0].isread = 1; + } else { + /* The host controller reads from memory */ + xfer->frbuffers[0].isread = 0; + } + } else { + /* default case */ + xfer->frbuffers[0].isread = isread; + } + + /* + * Setup the "page_start" pointer which points to an array of + * USB pages where information about the physical address of a + * page will be stored. Also initialise the "isread" field of + * the USB page caches. + */ + xfer->frbuffers[0].page_start = pg; + + info->dma_nframes = nframes; + info->dma_currframe = 0; + info->dma_frlength_0 = frlength_0; + + pg += (frlength_0 / USB_PAGE_SIZE); + pg += 2; + + while (--nframes > 0) { + xfer->frbuffers[nframes].isread = isread; + xfer->frbuffers[nframes].page_start = pg; + + pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); + pg += 2; + } + + } + if (info->dma_error) { + USB_BUS_LOCK(info->bus); + usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); + USB_BUS_UNLOCK(info->bus); + return; + } + if (info->dma_currframe != info->dma_nframes) { + + if (info->dma_currframe == 0) { + /* special case */ + usb2_pc_load_mem(xfer->frbuffers, + info->dma_frlength_0, 0); + } else { + /* default case */ + nframes = info->dma_currframe; + usb2_pc_load_mem(xfer->frbuffers + nframes, + xfer->frlengths[nframes], 0); + } + + /* advance frame index */ + info->dma_currframe++; + + return; + } + /* go ahead */ + usb2_bdma_pre_sync(xfer); + + /* start loading next USB transfer, if any */ + usb2_command_wrapper(pq, NULL); + + /* finally start the hardware */ + usb2_pipe_enter(xfer); +} + +/*------------------------------------------------------------------------* + * usb2_bdma_done_event + * + * This function is called when the BUS-DMA has loaded virtual memory + * into DMA, if any. + *------------------------------------------------------------------------*/ +void +usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt) +{ + struct usb2_xfer_root *info; + + info = udpt->info; + + mtx_assert(info->xfer_mtx, MA_OWNED); + + /* copy error */ + info->dma_error = udpt->dma_error; + + /* enter workloop again */ + usb2_command_wrapper(&info->dma_q, + info->dma_q.curr); +} + +/*------------------------------------------------------------------------* + * usb2_bdma_pre_sync + * + * This function handles DMA synchronisation that must be done before + * an USB transfer is started. + *------------------------------------------------------------------------*/ +void +usb2_bdma_pre_sync(struct usb2_xfer *xfer) +{ + struct usb2_page_cache *pc; + uint32_t nframes; + + if (xfer->flags_int.isochronous_xfr) { + /* only one frame buffer */ + nframes = 1; + } else { + /* can be multiple frame buffers */ + nframes = xfer->nframes; + } + + pc = xfer->frbuffers; + + while (nframes--) { + + if (pc->isread) { + usb2_pc_cpu_invalidate(pc); + } else { + usb2_pc_cpu_flush(pc); + } + pc++; + } +} + +/*------------------------------------------------------------------------* + * usb2_bdma_post_sync + * + * This function handles DMA synchronisation that must be done after + * an USB transfer is complete. + *------------------------------------------------------------------------*/ +void +usb2_bdma_post_sync(struct usb2_xfer *xfer) +{ + struct usb2_page_cache *pc; + uint32_t nframes; + + if (xfer->flags_int.isochronous_xfr) { + /* only one frame buffer */ + nframes = 1; + } else { + /* can be multiple frame buffers */ + nframes = xfer->nframes; + } + + pc = xfer->frbuffers; + + while (nframes--) { + if (pc->isread) { + usb2_pc_cpu_invalidate(pc); + } + pc++; + } +} |