aboutsummaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys')
-rw-r--r--sys/dev/ioat/ioat.c103
-rw-r--r--sys/dev/ioat/ioat_internal.h34
2 files changed, 86 insertions, 51 deletions
diff --git a/sys/dev/ioat/ioat.c b/sys/dev/ioat/ioat.c
index 6c96811cf0cc..fed6e37d380e 100644
--- a/sys/dev/ioat/ioat.c
+++ b/sys/dev/ioat/ioat.c
@@ -648,15 +648,18 @@ ioat_release(bus_dmaengine_t dmaengine)
mtx_unlock(&ioat->submit_lock);
}
-struct bus_dmadesc *
-ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
- void *callback_arg, uint32_t flags)
+static struct ioat_descriptor *
+ioat_op_generic(struct ioat_softc *ioat, uint8_t op,
+ uint32_t size, uint64_t src, uint64_t dst,
+ bus_dmaengine_callback_t callback_fn, void *callback_arg,
+ uint32_t flags)
{
- struct ioat_softc *ioat;
+ struct ioat_generic_hw_descriptor *hw_desc;
struct ioat_descriptor *desc;
- struct ioat_dma_hw_descriptor *hw_desc;
int mflags;
+ mtx_assert(&ioat->submit_lock, MA_OWNED);
+
KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
flags & ~DMA_ALL_FLAGS));
if ((flags & DMA_NO_WAIT) != 0)
@@ -664,31 +667,52 @@ ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
else
mflags = M_WAITOK;
- ioat = to_ioat_softc(dmaengine);
- mtx_assert(&ioat->submit_lock, MA_OWNED);
+ if (size > ioat->max_xfer_size) {
+ ioat_log_message(0, "%s: max_xfer_size = %d, requested = %u\n",
+ __func__, ioat->max_xfer_size, (unsigned)size);
+ return (NULL);
+ }
if (ioat_reserve_space(ioat, 1, mflags) != 0)
return (NULL);
- CTR0(KTR_IOAT, __func__);
-
desc = ioat_get_ring_entry(ioat, ioat->head);
- hw_desc = desc->u.dma;
+ hw_desc = desc->u.generic;
hw_desc->u.control_raw = 0;
- hw_desc->u.control.null = 1;
- hw_desc->u.control.completion_update = 1;
+ hw_desc->u.control_generic.op = op;
+ hw_desc->u.control_generic.completion_update = 1;
if ((flags & DMA_INT_EN) != 0)
- hw_desc->u.control.int_enable = 1;
+ hw_desc->u.control_generic.int_enable = 1;
- hw_desc->size = 8;
- hw_desc->src_addr = 0;
- hw_desc->dest_addr = 0;
+ hw_desc->size = size;
+ hw_desc->src_addr = src;
+ hw_desc->dest_addr = dst;
desc->bus_dmadesc.callback_fn = callback_fn;
desc->bus_dmadesc.callback_arg = callback_arg;
+ return (desc);
+}
+
+struct bus_dmadesc *
+ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
+ void *callback_arg, uint32_t flags)
+{
+ struct ioat_dma_hw_descriptor *hw_desc;
+ struct ioat_descriptor *desc;
+ struct ioat_softc *ioat;
+
+ CTR0(KTR_IOAT, __func__);
+ ioat = to_ioat_softc(dmaengine);
+ desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn,
+ callback_arg, flags);
+ if (desc == NULL)
+ return (NULL);
+
+ hw_desc = desc->u.dma;
+ hw_desc->u.control.null = 1;
ioat_submit_single(ioat);
return (&desc->bus_dmadesc);
}
@@ -698,51 +722,28 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn,
void *callback_arg, uint32_t flags)
{
- struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
+ struct ioat_descriptor *desc;
struct ioat_softc *ioat;
- int mflags;
-
- KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
- flags & ~DMA_ALL_FLAGS));
- if ((flags & DMA_NO_WAIT) != 0)
- mflags = M_NOWAIT;
- else
- mflags = M_WAITOK;
+ CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
- mtx_assert(&ioat->submit_lock, MA_OWNED);
- if (len > ioat->max_xfer_size) {
- ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n",
- __func__, ioat->max_xfer_size, (int)len);
+ if (((src | dst) & (0xffffull << 48)) != 0) {
+ ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n",
+ __func__);
return (NULL);
}
- if (ioat_reserve_space(ioat, 1, mflags) != 0)
+ desc = ioat_op_generic(ioat, IOAT_OP_COPY, len, src, dst, callback_fn,
+ callback_arg, flags);
+ if (desc == NULL)
return (NULL);
- CTR0(KTR_IOAT, __func__);
-
- desc = ioat_get_ring_entry(ioat, ioat->head);
hw_desc = desc->u.dma;
-
- hw_desc->u.control_raw = 0;
- hw_desc->u.control.completion_update = 1;
-
- if ((flags & DMA_INT_EN) != 0)
- hw_desc->u.control.int_enable = 1;
-
- hw_desc->size = len;
- hw_desc->src_addr = src;
- hw_desc->dest_addr = dst;
-
if (g_ioat_debug_level >= 3)
dump_descriptor(hw_desc);
- desc->bus_dmadesc.callback_fn = callback_fn;
- desc->bus_dmadesc.callback_arg = callback_arg;
-
ioat_submit_single(ioat);
return (&desc->bus_dmadesc);
}
@@ -767,7 +768,7 @@ ioat_get_ring_space(struct ioat_softc *ioat)
static struct ioat_descriptor *
ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
{
- struct ioat_dma_hw_descriptor *hw_desc;
+ struct ioat_generic_hw_descriptor *hw_desc;
struct ioat_descriptor *desc;
int error, busdmaflag;
@@ -788,7 +789,7 @@ ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
if (hw_desc == NULL)
goto out;
- desc->u.dma = hw_desc;
+ desc->u.generic = hw_desc;
error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr,
@@ -811,8 +812,8 @@ ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
if (desc == NULL)
return;
- if (desc->u.dma)
- bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
+ if (desc->u.generic)
+ bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic,
ioat->hw_desc_map);
free(desc, M_IOAT);
}
diff --git a/sys/dev/ioat/ioat_internal.h b/sys/dev/ioat/ioat_internal.h
index 87e52f13c6c1..2ca1ac65d805 100644
--- a/sys/dev/ioat/ioat_internal.h
+++ b/sys/dev/ioat/ioat_internal.h
@@ -122,10 +122,39 @@ SYSCTL_DECL(_hw_ioat);
extern int g_ioat_debug_level;
+struct generic_dma_control {
+ uint32_t int_enable:1;
+ uint32_t src_snoop_disable:1;
+ uint32_t dest_snoop_disable:1;
+ uint32_t completion_update:1;
+ uint32_t fence:1;
+ uint32_t reserved1:1;
+ uint32_t src_page_break:1;
+ uint32_t dest_page_break:1;
+ uint32_t bundle:1;
+ uint32_t dest_dca:1;
+ uint32_t hint:1;
+ uint32_t reserved2:13;
+ uint32_t op:8;
+};
+
+struct ioat_generic_hw_descriptor {
+ uint32_t size;
+ union {
+ uint32_t control_raw;
+ struct generic_dma_control control_generic;
+ } u;
+ uint64_t src_addr;
+ uint64_t dest_addr;
+ uint64_t next;
+ uint64_t reserved[4];
+};
+
struct ioat_dma_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
+ struct generic_dma_control control_generic;
struct {
uint32_t int_enable:1;
uint32_t src_snoop_disable:1;
@@ -156,6 +185,7 @@ struct ioat_fill_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
+ struct generic_dma_control control_generic;
struct {
uint32_t int_enable:1;
uint32_t reserved:1;
@@ -183,6 +213,7 @@ struct ioat_xor_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
+ struct generic_dma_control control_generic;
struct {
uint32_t int_enable:1;
uint32_t src_snoop_disable:1;
@@ -220,6 +251,7 @@ struct ioat_pq_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
+ struct generic_dma_control control_generic;
struct {
uint32_t int_enable:1;
uint32_t src_snoop_disable:1;
@@ -261,6 +293,7 @@ struct ioat_pq_update_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
+ struct generic_dma_control control_generic;
struct {
uint32_t int_enable:1;
uint32_t src_snoop_disable:1;
@@ -300,6 +333,7 @@ struct bus_dmadesc {
struct ioat_descriptor {
struct bus_dmadesc bus_dmadesc;
union {
+ struct ioat_generic_hw_descriptor *generic;
struct ioat_dma_hw_descriptor *dma;
struct ioat_fill_hw_descriptor *fill;
struct ioat_xor_hw_descriptor *xor;
generated by cgit v1.2.3 (git 2.25.1) at 2023-09-29 08:25:38 +0000