Generalize AES iov optimization

Right now, aesni_cipher_alloc does a bit of special-casing
for CRYPTO_F_IOV, to not do any allocation if the first uio
is large enough for the requested size. While working on ZFS
crypto port, I ran into horrible performance because the code
uses scatter-gather, and many of the times the data to encrypt
was in the second entry. This code looks through the list, and
tries to see if there is a single uio that can contain the
requested data, and, if so, uses that.

This has a slight impact on the current consumers, in that the
check is a little more complicated for the ones that use
CRYPTO_F_IOV -- but none of them meet the criteria for testing
more than one.

Submitted by:	sef at ixsystems.com
Reviewed by:	cem@
MFC after:	3 days
Sponsored by:	iX Systems
Differential Revision:	https://reviews.freebsd.org/D18522

1 files changed, 54 insertions, 0 deletions

diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c
index 71e395a910e0..d80894909130 100644
--- a/sys/opencrypto/criov.c
+++ b/sys/opencrypto/criov.c
@@ -38,6 +38,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/uio.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
 
 #include <opencrypto/cryptodev.h>
 
@@ -239,3 +241,55 @@ crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, int *cnt,
 	*cnt = i;
 	return 0;
 }
+
+static inline void *
+m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
+{
+	int rel_off;
+
+	MPASS(skip <= INT_MAX);
+
+	m = m_getptr(m, (int)skip, &rel_off);
+	if (m == NULL)
+		return (NULL);
+
+	MPASS(rel_off >= 0);
+	skip = rel_off;
+	if (skip + len > m->m_len)
+		return (NULL);
+
+	return (mtod(m, char*) + skip);
+}
+
+static inline void *
+cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
+{
+	int rel_off, idx;
+
+	MPASS(skip <= INT_MAX);
+	idx = cuio_getptr(uio, (int)skip, &rel_off);
+	if (idx < 0)
+		return (NULL);
+
+	MPASS(rel_off >= 0);
+	skip = rel_off;
+	if (skip + len > uio->uio_iov[idx].iov_len)
+		return (NULL);
+	return ((char *)uio->uio_iov[idx].iov_base + skip);
+}
+
+void *
+crypto_contiguous_subsegment(int crp_flags, void *crpbuf,
+    size_t skip, size_t len)
+{
+	if ((crp_flags & CRYPTO_F_IMBUF) != 0)
+		return (m_contiguous_subsegment(crpbuf, skip, len));
+	else if ((crp_flags & CRYPTO_F_IOV) != 0)
+		return (cuio_contiguous_segment(crpbuf, skip, len));
+	else {
+		MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) !=
+		    (CRYPTO_F_IMBUF | CRYPTO_F_IOV));
+		return ((char*)crpbuf + skip);
+	}
+}
+