aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlan Somers <asomers@FreeBSD.org>2020-08-26 02:37:42 +0000
committerAlan Somers <asomers@FreeBSD.org>2020-08-26 02:37:42 +0000
commite6f6d0c9bcbf7942c390f65062054ec4784ce5b8 (patch)
treea83b50c9e02f47cd1b8d6091a31f2398db82bd19
parentd4d2928a3fc688c04d9a03c8a95bc96537d2c64b (diff)
downloadsrc-e6f6d0c9bcbf7942c390f65062054ec4784ce5b8.tar.gz
src-e6f6d0c9bcbf7942c390f65062054ec4784ce5b8.zip
crypto(9): add CRYPTO_BUF_VMPAGE
crypto(9) functions can now be used on buffers composed of an array of vm_page_t structures, such as those stored in an unmapped struct bio. It requires the running to kernel to support the direct memory map, so not all architectures can use it. Reviewed by: markj, kib, jhb, mjg, mat, bcr (manpages) MFC after: 1 week Sponsored by: Axcient Differential Revision: https://reviews.freebsd.org/D25671
Notes
Notes: svn path=/head/; revision=364799
-rw-r--r--share/man/man9/crypto_buffer.924
-rw-r--r--share/man/man9/crypto_request.918
-rw-r--r--sys/crypto/ccp/ccp.c4
-rw-r--r--sys/dev/cxgbe/crypto/t4_crypto.c4
-rw-r--r--sys/dev/sec/sec.c3
-rw-r--r--sys/kern/subr_bus_dma.c5
-rw-r--r--sys/opencrypto/criov.c215
-rw-r--r--sys/opencrypto/crypto.c22
-rw-r--r--sys/opencrypto/cryptodev.h52
-rw-r--r--sys/opencrypto/cryptosoft.c4
10 files changed, 338 insertions, 13 deletions
diff --git a/share/man/man9/crypto_buffer.9 b/share/man/man9/crypto_buffer.9
index 9ba77b860636..2f39c21f9214 100644
--- a/share/man/man9/crypto_buffer.9
+++ b/share/man/man9/crypto_buffer.9
@@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd May 25, 2020
+.Dd August 12, 2020
.Dt CRYPTO_BUFFER 9
.Os
.Sh NAME
@@ -197,10 +197,17 @@ A scatter/gather list of kernel buffers as described in
.It Dv CRYPTO_BUF_MBUF
A network memory buffer as described in
.Xr mbuf 9 .
+.It Dv CRYPTO_BUF_VMPAGE
+A scatter/gather list of
+.Vt vm_page_t
+structures describing pages in the kernel's address space.
+This buffer type is only available if
+.Dv CRYPTO_HAS_VMPAGE
+is true.
.El
.Pp
The structure also contains the following type-specific fields:
-.Bl -tag -width " cb_buf_len"
+.Bl -tag -width " cb_vm_page_offset"
.It Fa cb_buf
A pointer to the start of a
.Dv CRYPTO_BUF_CONTIG
@@ -219,6 +226,19 @@ A pointer to a
.Vt struct uio
for
.Dv CRYPTO_BUF_UIO .
+.It Fa cb_vm_page
+A pointer to an array of
+.Vt struct vm_page
+for
+.Dv CRYPTO_BUF_VMPAGE .
+.It Fa cb_vm_page_len
+The total amount of data included in the
+.Fa cb_vm_page
+array, in bytes.
+.It Fa cb_vm_page_offset
+Offset in bytes in the first page of
+.Fa cb_vm_page
+where valid data begins.
.El
.Ss Cursors
Cursors provide a mechanism for iterating over a data buffer.
diff --git a/share/man/man9/crypto_request.9 b/share/man/man9/crypto_request.9
index 69b2a82d7ab1..8ba075b06cf0 100644
--- a/share/man/man9/crypto_request.9
+++ b/share/man/man9/crypto_request.9
@@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd July 16, 2020
+.Dd August 12, 2020
.Dt CRYPTO_REQUEST 9
.Os
.Sh NAME
@@ -55,11 +55,15 @@
.Ft void
.Fn crypto_use_uio "struct cryptop *crp" "struct uio *uio"
.Ft void
+.Fn crypto_use_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset"
+.Ft void
.Fn crypto_use_output_buf "struct cryptop *crp" "void *buf" "int len"
.Ft void
.Fn crypto_use_output_mbuf "struct cryptop *crp" "struct mbuf *m"
.Ft void
.Fn crypto_use_output_uio "struct cryptop *crp" "struct uio *uio"
+.Ft void
+.Fn crypto_use_output_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset"
.Sh DESCRIPTION
Each symmetric cryptographic operation in the kernel is described by
an instance of
@@ -141,7 +145,7 @@ mode requests.
All requests must have a valid
.Fa crp_buf
initialized by one of the following functions:
-.Bl -tag -width "Fn crypto_use_mbuf"
+.Bl -tag -width "Fn crypto_use_vmpage"
.It Fn crypto_use_buf
Uses an array of
.Fa len
@@ -156,12 +160,16 @@ as the data buffer.
Uses the scatter/gather list
.Fa uio
as the data buffer.
+.It Fn crypto_use_vmpage
+Uses the array of
+.Vt vm_page_t
+structures as the data buffer.
.El
.Pp
One of the following functions should be used to initialize
.Fa crp_obuf
for requests that use separate input and output buffers:
-.Bl -tag -width "Fn crypto_use_output_mbuf"
+.Bl -tag -width "Fn crypto_use_output_vmpage"
.It Fn crypto_use_output_buf
Uses an array of
.Fa len
@@ -176,6 +184,10 @@ as the output buffer.
Uses the scatter/gather list
.Fa uio
as the output buffer.
+.It Fn crypto_use_output_vmpage
+Uses the array of
+.Vt vm_page_t
+structures as the output buffer.
.El
.Ss Request Regions
Each request describes one or more regions in the data buffers.
diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c
index f569d18caa54..63967d7f8e3c 100644
--- a/sys/crypto/ccp/ccp.c
+++ b/sys/crypto/ccp/ccp.c
@@ -107,6 +107,10 @@ ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
break;
+ case CRYPTO_BUF_VMPAGE:
+ error = sglist_append_vmpages(sg, cb->cb_vm_page,
+ cb->cb_vm_page_len, cb->cb_vm_page_offset);
+ break;
default:
error = EINVAL;
}
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c
index 024883028943..ff2af644d60b 100644
--- a/sys/dev/cxgbe/crypto/t4_crypto.c
+++ b/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -272,6 +272,10 @@ ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
break;
+ case CRYPTO_BUF_VMPAGE:
+ error = sglist_append_vmpages(sg, cb->cb_vm_page,
+ cb->cb_vm_page_len, cb->cb_vm_page_offset);
+ break;
default:
error = EINVAL;
}
diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c
index a3c4fdac3de9..342ee849aa2b 100644
--- a/sys/dev/sec/sec.c
+++ b/sys/dev/sec/sec.c
@@ -851,6 +851,9 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
case CRYPTO_BUF_MBUF:
size = m_length(crp->crp_buf.cb_mbuf, NULL);
break;
+ case CRYPTO_BUF_VMPAGE:
+ size = PAGE_SIZE - cb->cb_vm_page_offset;
+ break;
default:
return (EINVAL);
}
diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c
index 896bd8c1c346..a0a2287855f9 100644
--- a/sys/kern/subr_bus_dma.c
+++ b/sys/kern/subr_bus_dma.c
@@ -661,6 +661,11 @@ bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map,
error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs,
flags);
break;
+ case CRYPTO_BUF_VMPAGE:
+ error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page,
+ cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL,
+ &nsegs);
+ break;
default:
error = EINVAL;
}
diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c
index 0ef39eac1eb2..b18a01faf26a 100644
--- a/sys/opencrypto/criov.c
+++ b/sys/opencrypto/criov.c
@@ -40,12 +40,21 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <sys/limits.h>
#include <sys/lock.h>
+#include <sys/sdt.h>
+
+#include <machine/vmparam.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/pmap.h>
#include <opencrypto/cryptodev.h>
+SDT_PROVIDER_DECLARE(opencrypto);
+
/*
- * This macro is only for avoiding code duplication, as we need to skip
- * given number of bytes in the same way in three functions below.
+ * These macros are only for avoiding code duplication, as we need to skip
+ * given number of bytes in the same way in several functions below.
*/
#define CUIO_SKIP() do { \
KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
@@ -60,6 +69,18 @@ __FBSDID("$FreeBSD$");
} \
} while (0)
+#define CVM_PAGE_SKIP() do { \
+ KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
+ KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
+ while (off > 0) { \
+ if (off < PAGE_SIZE) \
+ break; \
+ processed += PAGE_SIZE - off; \
+ off -= PAGE_SIZE - off; \
+ pages++; \
+ } \
+} while (0)
+
static void
cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
{
@@ -128,6 +149,96 @@ cuio_getptr(struct uio *uio, int loc, int *off)
return (-1);
}
+#if CRYPTO_MAY_HAVE_VMPAGE
+/*
+ * Apply function f to the data in a vm_page_t list starting "off" bytes from
+ * the beginning, continuing for "len" bytes.
+ */
+static int
+cvm_page_apply(vm_page_t *pages, int off, int len,
+ int (*f)(void *, const void *, u_int), void *arg)
+{
+ int processed = 0;
+ unsigned count;
+ int rval;
+
+ CVM_PAGE_SKIP();
+ while (len > 0) {
+ char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages));
+ count = min(PAGE_SIZE - off, len);
+ rval = (*f)(arg, kaddr + off, count);
+ if (rval)
+ return (rval);
+ len -= count;
+ processed += count;
+ off = 0;
+ pages++;
+ }
+ return (0);
+}
+
+static inline void *
+cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
+{
+ if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
+ return (NULL);
+
+ pages += (skip / PAGE_SIZE);
+ skip -= rounddown(skip, PAGE_SIZE);
+ return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip);
+}
+
+/*
+ * Copy len bytes of data from the vm_page_t array, skipping the first off
+ * bytes, into the pointer cp. Return the number of bytes skipped and copied.
+ * Does not verify the length of the array.
+ */
+static int
+cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
+{
+ int processed = 0;
+ unsigned count;
+
+ CVM_PAGE_SKIP();
+ while (len > 0) {
+ count = min(PAGE_SIZE - off, len);
+ bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off,
+ count);
+ len -= count;
+ cp += count;
+ processed += count;
+ off = 0;
+ pages++;
+ }
+ return (processed);
+}
+
+/*
+ * Copy len bytes of data from the pointer cp into the vm_page_t array,
+ * skipping the first off bytes, Return the number of bytes skipped and copied.
+ * Does not verify the length of the array.
+ */
+static int
+cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
+{
+ int processed = 0;
+ unsigned count;
+
+ CVM_PAGE_SKIP();
+ while (len > 0) {
+ count = min(PAGE_SIZE - off, len);
+ bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp,
+ count);
+ len -= count;
+ cp += count;
+ processed += count;
+ off = 0;
+ pages++;
+ }
+ return processed;
+}
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
+
void
crypto_cursor_init(struct crypto_buffer_cursor *cc,
const struct crypto_buffer *cb)
@@ -142,6 +253,11 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
case CRYPTO_BUF_MBUF:
cc->cc_mbuf = cb->cb_mbuf;
break;
+ case CRYPTO_BUF_VMPAGE:
+ cc->cc_vmpage = cb->cb_vm_page;
+ cc->cc_buf_len = cb->cb_vm_page_len;
+ cc->cc_offset = cb->cb_vm_page_offset;
+ break;
case CRYPTO_BUF_UIO:
cc->cc_iov = cb->cb_uio->uio_iov;
break;
@@ -153,6 +269,8 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
}
}
+SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
+
void
crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
{
@@ -178,6 +296,24 @@ crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
break;
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ for (;;) {
+ SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
+ cc, amount);
+ remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
+ if (amount < remain) {
+ cc->cc_buf_len -= amount;
+ cc->cc_offset += amount;
+ break;
+ }
+ cc->cc_buf_len -= remain;
+ amount -= remain;
+ cc->cc_vmpage++;
+ cc->cc_offset = 0;
+ if (amount == 0 || cc->cc_buf_len == 0)
+ break;
+ }
+ break;
case CRYPTO_BUF_UIO:
for (;;) {
remain = cc->cc_iov->iov_len - cc->cc_offset;
@@ -212,6 +348,9 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
("%s: not supported for unmapped mbufs", __func__));
return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
+ case CRYPTO_BUF_VMPAGE:
+ return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
+ *cc->cc_vmpage)) + cc->cc_offset);
case CRYPTO_BUF_UIO:
return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
default:
@@ -228,6 +367,8 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
switch (cc->cc_type) {
case CRYPTO_BUF_CONTIG:
return (cc->cc_buf_len);
+ case CRYPTO_BUF_VMPAGE:
+ return (PAGE_SIZE - cc->cc_offset);
case CRYPTO_BUF_MBUF:
if (cc->cc_mbuf == NULL)
return (0);
@@ -278,6 +419,26 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
break;
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ for (;;) {
+ dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
+ *cc->cc_vmpage)) + cc->cc_offset;
+ remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
+ todo = MIN(remain, size);
+ memcpy(dst, src, todo);
+ src += todo;
+ cc->cc_buf_len -= todo;
+ if (todo < remain) {
+ cc->cc_offset += todo;
+ break;
+ }
+ size -= todo;
+ cc->cc_vmpage++;
+ cc->cc_offset = 0;
+ if (size == 0)
+ break;
+ }
+ break;
case CRYPTO_BUF_UIO:
for (;;) {
dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
@@ -339,6 +500,26 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
break;
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ for (;;) {
+ src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
+ *cc->cc_vmpage)) + cc->cc_offset;
+ remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
+ todo = MIN(remain, size);
+ memcpy(dst, src, todo);
+ src += todo;
+ cc->cc_buf_len -= todo;
+ if (todo < remain) {
+ cc->cc_offset += todo;
+ break;
+ }
+ size -= todo;
+ cc->cc_vmpage++;
+ cc->cc_offset = 0;
+ if (size == 0)
+ break;
+ }
+ break;
case CRYPTO_BUF_UIO:
for (;;) {
src = (const char *)cc->cc_iov->iov_base +
@@ -421,6 +602,15 @@ crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
case CRYPTO_BUF_MBUF:
m_copyback(cb->cb_mbuf, off, size, src);
break;
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ MPASS(size <= cb->cb_vm_page_len);
+ MPASS(size + off <=
+ cb->cb_vm_page_len + cb->cb_vm_page_offset);
+ cvm_page_copyback(cb->cb_vm_page,
+ off + cb->cb_vm_page_offset, size, src);
+ break;
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_UIO:
cuio_copyback(cb->cb_uio, off, size, src);
break;
@@ -444,6 +634,15 @@ crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
case CRYPTO_BUF_MBUF:
m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
break;
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ MPASS(size <= crp->crp_buf.cb_vm_page_len);
+ MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
+ crp->crp_buf.cb_vm_page_offset);
+ cvm_page_copydata(crp->crp_buf.cb_vm_page,
+ off + crp->crp_buf.cb_vm_page_offset, size, dst);
+ break;
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_UIO:
cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
break;
@@ -473,6 +672,12 @@ crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
case CRYPTO_BUF_UIO:
error = cuio_apply(cb->cb_uio, off, len, f, arg);
break;
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ error = cvm_page_apply(cb->cb_vm_page,
+ off + cb->cb_vm_page_offset, len, f, arg);
+ break;
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_CONTIG:
MPASS(off + len <= cb->cb_buf_len);
error = (*f)(arg, cb->cb_buf + off, len);
@@ -540,6 +745,12 @@ crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
case CRYPTO_BUF_UIO:
return (cuio_contiguous_segment(cb->cb_uio, skip, len));
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ MPASS(skip + len <= cb->cb_vm_page_len);
+ return (cvm_page_contiguous_segment(cb->cb_vm_page,
+ skip + cb->cb_vm_page_offset, len));
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_CONTIG:
MPASS(skip + len <= cb->cb_buf_len);
return (cb->cb_buf + skip);
diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
index c7bf240b0fb7..b4e0f06f3588 100644
--- a/sys/opencrypto/crypto.c
+++ b/sys/opencrypto/crypto.c
@@ -78,7 +78,9 @@ __FBSDID("$FreeBSD$");
#include <ddb/ddb.h>
+#include <machine/vmparam.h>
#include <vm/uma.h>
+
#include <crypto/intake.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform_auth.h>
@@ -1218,6 +1220,8 @@ crypto_buffer_len(struct crypto_buffer *cb)
if (cb->cb_mbuf->m_flags & M_PKTHDR)
return (cb->cb_mbuf->m_pkthdr.len);
return (m_length(cb->cb_mbuf, NULL));
+ case CRYPTO_BUF_VMPAGE:
+ return (cb->cb_vm_page_len);
case CRYPTO_BUF_UIO:
return (cb->cb_uio->uio_resid);
default:
@@ -1232,9 +1236,25 @@ cb_sanity(struct crypto_buffer *cb, const char *name)
{
KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
("incoming crp with invalid %s buffer type", name));
- if (cb->cb_type == CRYPTO_BUF_CONTIG)
+ switch (cb->cb_type) {
+ case CRYPTO_BUF_CONTIG:
KASSERT(cb->cb_buf_len >= 0,
("incoming crp with -ve %s buffer length", name));
+ break;
+ case CRYPTO_BUF_VMPAGE:
+ KASSERT(CRYPTO_HAS_VMPAGE,
+ ("incoming crp uses dmap on supported arch"));
+ KASSERT(cb->cb_vm_page_len >= 0,
+ ("incoming crp with -ve %s buffer length", name));
+ KASSERT(cb->cb_vm_page_offset >= 0,
+ ("incoming crp with -ve %s buffer offset", name));
+ KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
+ ("incoming crp with %s buffer offset greater than page size"
+ , name));
+ break;
+ default:
+ break;
+ }
}
static void
diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h
index edde16cf7327..2f2963edcb51 100644
--- a/sys/opencrypto/cryptodev.h
+++ b/sys/opencrypto/cryptodev.h
@@ -205,6 +205,15 @@
#define CRYPTO_FLAG_HARDWARE 0x01000000 /* hardware accelerated */
#define CRYPTO_FLAG_SOFTWARE 0x02000000 /* software implementation */
+/* Does the kernel support vmpage buffers on this platform? */
+#ifdef __powerpc__
+#define CRYPTO_MAY_HAVE_VMPAGE 1
+#else
+#define CRYPTO_MAY_HAVE_VMPAGE ( PMAP_HAS_DMAP )
+#endif
+/* Does the currently running system support vmpage buffers on this platform? */
+#define CRYPTO_HAS_VMPAGE ( PMAP_HAS_DMAP )
+
/* NB: deprecated */
struct session_op {
u_int32_t cipher; /* ie. CRYPTO_AES_CBC */
@@ -387,7 +396,8 @@ enum crypto_buffer_type {
CRYPTO_BUF_CONTIG,
CRYPTO_BUF_UIO,
CRYPTO_BUF_MBUF,
- CRYPTO_BUF_LAST = CRYPTO_BUF_MBUF
+ CRYPTO_BUF_VMPAGE,
+ CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE
};
/*
@@ -402,6 +412,11 @@ struct crypto_buffer {
int cb_buf_len;
};
struct mbuf *cb_mbuf;
+ struct {
+ vm_page_t *cb_vm_page;
+ int cb_vm_page_len;
+ int cb_vm_page_offset;
+ };
struct uio *cb_uio;
};
enum crypto_buffer_type cb_type;
@@ -415,11 +430,15 @@ struct crypto_buffer_cursor {
char *cc_buf;
struct mbuf *cc_mbuf;
struct iovec *cc_iov;
+ vm_page_t *cc_vmpage;
};
- union {
- int cc_buf_len;
- size_t cc_offset;
- };
+ /* Optional bytes of valid data remaining */
+ int cc_buf_len;
+ /*
+ * Optional offset within the current buffer segment where
+ * valid data begins
+ */
+ size_t cc_offset;
enum crypto_buffer_type cc_type;
};
@@ -509,6 +528,16 @@ _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m)
}
static __inline void
+_crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len,
+ int offset)
+{
+ cb->cb_vm_page = pages;
+ cb->cb_vm_page_len = len;
+ cb->cb_vm_page_offset = offset;
+ cb->cb_type = CRYPTO_BUF_VMPAGE;
+}
+
+static __inline void
_crypto_use_uio(struct crypto_buffer *cb, struct uio *uio)
{
cb->cb_uio = uio;
@@ -528,6 +557,12 @@ crypto_use_mbuf(struct cryptop *crp, struct mbuf *m)
}
static __inline void
+crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset)
+{
+ _crypto_use_vmpage(&crp->crp_buf, pages, len, offset);
+}
+
+static __inline void
crypto_use_uio(struct cryptop *crp, struct uio *uio)
{
_crypto_use_uio(&crp->crp_buf, uio);
@@ -546,6 +581,13 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
}
static __inline void
+crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len,
+ int offset)
+{
+ _crypto_use_vmpage(&crp->crp_obuf, pages, len, offset);
+}
+
+static __inline void
crypto_use_output_uio(struct cryptop *crp, struct uio *uio)
{
_crypto_use_uio(&crp->crp_obuf, uio);
diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c
index 27a6d472e3d8..0041e96ac0dc 100644
--- a/sys/opencrypto/cryptosoft.c
+++ b/sys/opencrypto/cryptosoft.c
@@ -980,6 +980,10 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
}
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ adj = crp->crp_payload_length - result;
+ crp->crp_buf.cb_vm_page_len -= adj;
+ break;
default:
break;
}