armv8crypto: Use cursors to access crypto buffer data

Currently armv8crypto copies the scheme used in aesni(9), where payload
data and output buffers are allocated on the fly if the crypto buffer is
not virtually contiguous.  This scheme is simple but incurs a lot of
overhead: for an encryption request with a separate output buffer we
have to
- allocate a temporary buffer to hold the payload
- copy input data into the buffer
- copy the encrypted payload to the output buffer
- zero the temporary buffer before freeing it

We have a handy crypto buffer cursor abstraction now, so reimplement the
armv8crypto routines using that instead of temporary buffers.  This
introduces some extra complexity, but gallatin@ reports a 10% throughput
improvement with a KTLS workload without additional CPU usage.  The
driver still allocates an AAD buffer for AES-GCM if necessary.

Reviewed by:	jhb
Tested by:	gallatin
Sponsored by:	Ampere Computing LLC
Submitted by:	Klara Inc.
MFC after:	1 month
Differential Revision:	https://reviews.freebsd.org/D28950

3 files changed, 266 insertions, 171 deletions

diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c
index 0811a1c03390..18b0870f380b 100644
--- a/sys/crypto/armv8/armv8_crypto.c
+++ b/sys/crypto/armv8/armv8_crypto.c
@@ -378,30 +378,23 @@ static int
 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
     struct cryptop *crp)
 {
+	struct crypto_buffer_cursor fromc, toc;
 	const struct crypto_session_params *csp;
 	struct fpu_kern_ctx *ctx;
-	uint8_t *buf, *authbuf, *outbuf;
+	uint8_t *authbuf;
 	uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
-	int allocated, authallocated, outallocated, i;
+	int authallocated, i;
 	int encflag;
 	int kt;
 	int error;
-	bool outcopy;
 
 	csp = crypto_get_params(crp->crp_session);
 	encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
 
-	allocated = 0;
-	outallocated = 0;
 	authallocated = 0;
 	authbuf = NULL;
 	kt = 1;
 
-	buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start,
-	    crp->crp_payload_length, &allocated);
-	if (buf == NULL)
-		return (ENOMEM);
-
 	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
 		if (crp->crp_aad != NULL)
 			authbuf = crp->crp_aad;
@@ -413,28 +406,13 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 			goto out;
 		}
 	}
-
+	crypto_cursor_init(&fromc, &crp->crp_buf);
+	crypto_cursor_advance(&fromc, crp->crp_payload_start);
 	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
-		outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
-		    crp->crp_payload_output_start, crp->crp_payload_length);
-		if (outbuf == NULL) {
-			outcopy = true;
-			if (allocated)
-				outbuf = buf;
-			else {
-				outbuf = malloc(crp->crp_payload_length,
-				    M_ARMV8_CRYPTO, M_NOWAIT);
-				if (outbuf == NULL) {
-					error = ENOMEM;
-					goto out;
-				}
-				outallocated = true;
-			}
-		} else
-			outcopy = false;
+		crypto_cursor_init(&toc, &crp->crp_obuf);
+		crypto_cursor_advance(&toc, crp->crp_payload_output_start);
 	} else {
-		outbuf = buf;
-		outcopy = allocated;
+		crypto_cursor_copy(&fromc, &toc);
 	}
 
 	kt = is_fpu_kern_thread(0);
@@ -451,7 +429,6 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 
 	crypto_read_iv(crp, iv);
 
-	/* Do work */
 	switch (csp->csp_cipher_alg) {
 	case CRYPTO_AES_CBC:
 		if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
@@ -460,51 +437,41 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 		}
 		if (encflag)
 			armv8_aes_encrypt_cbc(&ses->enc_schedule,
-			    crp->crp_payload_length, buf, buf, iv);
+			    crp->crp_payload_length, &fromc, &toc, iv);
 		else
 			armv8_aes_decrypt_cbc(&ses->dec_schedule,
-			    crp->crp_payload_length, buf, iv);
+			    crp->crp_payload_length, &fromc, &toc, iv);
 		break;
 	case CRYPTO_AES_XTS:
 		if (encflag)
 			armv8_aes_encrypt_xts(&ses->enc_schedule,
-			    &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
-			    buf, iv);
+			    &ses->xts_schedule.aes_key, crp->crp_payload_length,
+			    &fromc, &toc, iv);
 		else
 			armv8_aes_decrypt_xts(&ses->dec_schedule,
-			    &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
-			    buf, iv);
+			    &ses->xts_schedule.aes_key, crp->crp_payload_length,
+			    &fromc, &toc, iv);
 		break;
 	case CRYPTO_AES_NIST_GCM_16:
 		if (encflag) {
 			memset(tag, 0, sizeof(tag));
 			armv8_aes_encrypt_gcm(&ses->enc_schedule,
-			    crp->crp_payload_length,
-			    buf, outbuf,
-			    crp->crp_aad_length, authbuf,
-			    tag, iv, ses->Htable);
+			    crp->crp_payload_length, &fromc, &toc,
+			    crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
 			crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
 			    tag);
 		} else {
 			crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
 			    tag);
-			if (armv8_aes_decrypt_gcm(&ses->enc_schedule,
-			    crp->crp_payload_length,
-			    buf, outbuf,
-			    crp->crp_aad_length, authbuf,
-			    tag, iv, ses->Htable) != 0) {
-				error = EBADMSG;
+			error = armv8_aes_decrypt_gcm(&ses->enc_schedule,
+			    crp->crp_payload_length, &fromc, &toc,
+			    crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
+			if (error != 0)
 				goto out;
-			}
 		}
 		break;
 	}
 
-	if (outcopy)
-		crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
-		    crp->crp_payload_output_start : crp->crp_payload_start,
-		    crp->crp_payload_length, outbuf);
-
 	error = 0;
 out:
 	if (!kt) {
@@ -512,12 +479,8 @@ out:
 		RELEASE_CTX(i, ctx);
 	}
 
-	if (allocated)
-		zfree(buf, M_ARMV8_CRYPTO);
 	if (authallocated)
 		zfree(authbuf, M_ARMV8_CRYPTO);
-	if (outallocated)
-		zfree(outbuf, M_ARMV8_CRYPTO);
 	explicit_bzero(iv, sizeof(iv));
 	explicit_bzero(tag, sizeof(tag));
 
diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h
index 855aabd8bac3..a978248703fe 100644
--- a/sys/crypto/armv8/armv8_crypto.h
+++ b/sys/crypto/armv8/armv8_crypto.h
@@ -63,24 +63,29 @@ void gcm_init_v8(__uint128_val_t Htable[16], const uint64_t Xi[2]);
 void gcm_gmult_v8(uint64_t Xi[2], const __uint128_val_t Htable[16]);
 void gcm_ghash_v8(uint64_t Xi[2], const __uint128_val_t Htable[16], const uint8_t *inp, size_t len);
 
-void armv8_aes_encrypt_cbc(const AES_key_t *, size_t, const uint8_t *,
-    uint8_t *, const uint8_t[static AES_BLOCK_LEN]);
-void armv8_aes_decrypt_cbc(const AES_key_t *, size_t, uint8_t *,
+void armv8_aes_encrypt_cbc(const AES_key_t *key, size_t len,
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
+    const uint8_t iv[static AES_BLOCK_LEN]);
+void armv8_aes_decrypt_cbc(const AES_key_t *, size_t,
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
     const uint8_t[static AES_BLOCK_LEN]);
-void armv8_aes_encrypt_gcm(AES_key_t *, size_t, const uint8_t *,
-    uint8_t *, size_t, const uint8_t*,
+void armv8_aes_encrypt_gcm(AES_key_t *, size_t,
+    struct crypto_buffer_cursor *, struct crypto_buffer_cursor *,
+    size_t, const uint8_t *,
     uint8_t tag[static GMAC_DIGEST_LEN],
     const uint8_t[static AES_BLOCK_LEN],
     const __uint128_val_t *);
-int armv8_aes_decrypt_gcm(AES_key_t *, size_t, const uint8_t *,
-    uint8_t *, size_t, const uint8_t*,
-    const uint8_t tag[static GMAC_DIGEST_LEN],
+int armv8_aes_decrypt_gcm(AES_key_t *, size_t,
+    struct crypto_buffer_cursor *, struct crypto_buffer_cursor *,
+    size_t, const uint8_t *, const uint8_t tag[static GMAC_DIGEST_LEN],
     const uint8_t[static AES_BLOCK_LEN],
     const __uint128_val_t *);
 
 void armv8_aes_encrypt_xts(AES_key_t *, const void *, size_t,
-    const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
+    struct crypto_buffer_cursor *, struct crypto_buffer_cursor *,
+    const uint8_t[AES_BLOCK_LEN]);
 void armv8_aes_decrypt_xts(AES_key_t *, const void *, size_t,
-    const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
+    struct crypto_buffer_cursor *, struct crypto_buffer_cursor *,
+    const uint8_t[AES_BLOCK_LEN]);
 
 #endif /* _ARMV8_CRYPTO_H_ */
diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c
index b5aee0cc1cf6..8c489978eaaa 100644
--- a/sys/crypto/armv8/armv8_crypto_wrap.c
+++ b/sys/crypto/armv8/armv8_crypto_wrap.c
@@ -101,41 +101,97 @@ armv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
 
 void
 armv8_aes_encrypt_cbc(const AES_key_t *key, size_t len,
-    const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
+    const uint8_t iv[static AES_BLOCK_LEN])
 {
 	uint8x16_t tot, ivreg, tmp;
-	size_t i;
+	uint8_t block[AES_BLOCK_LEN], *from, *to;
+	size_t fromseglen, oseglen, seglen, toseglen;
+
+	KASSERT(len % AES_BLOCK_LEN == 0,
+	    ("%s: length %zu not a multiple of the block size", __func__, len));
 
-	len /= AES_BLOCK_LEN;
 	ivreg = vld1q_u8(iv);
-	for (i = 0; i < len; i++) {
-		tmp = vld1q_u8(from);
-		tot = armv8_aes_enc(key->aes_rounds - 1,
-		    (const void*)key->aes_key, veorq_u8(tmp, ivreg));
-		ivreg = tot;
-		vst1q_u8(to, tot);
-		from += AES_BLOCK_LEN;
-		to += AES_BLOCK_LEN;
+	for (; len > 0; len -= seglen) {
+		from = crypto_cursor_segment(fromc, &fromseglen);
+		to = crypto_cursor_segment(toc, &toseglen);
+
+		seglen = ulmin(len, ulmin(fromseglen, toseglen));
+		if (seglen < AES_BLOCK_LEN) {
+			crypto_cursor_copydata(fromc, AES_BLOCK_LEN, block);
+			tmp = vld1q_u8(block);
+			tot = armv8_aes_enc(key->aes_rounds - 1,
+			    (const void *)key->aes_key, veorq_u8(tmp, ivreg));
+			ivreg = tot;
+			vst1q_u8(block, tot);
+			crypto_cursor_copyback(toc, AES_BLOCK_LEN, block);
+			seglen = AES_BLOCK_LEN;
+		} else {
+			for (oseglen = seglen; seglen >= AES_BLOCK_LEN;
+			    seglen -= AES_BLOCK_LEN) {
+				tmp = vld1q_u8(from);
+				tot = armv8_aes_enc(key->aes_rounds - 1,
+				    (const void *)key->aes_key,
+				    veorq_u8(tmp, ivreg));
+				ivreg = tot;
+				vst1q_u8(to, tot);
+				from += AES_BLOCK_LEN;
+				to += AES_BLOCK_LEN;
+			}
+			seglen = oseglen - seglen;
+			crypto_cursor_advance(fromc, seglen);
+			crypto_cursor_advance(toc, seglen);
+		}
 	}
+
+	explicit_bzero(block, sizeof(block));
 }
 
 void
 armv8_aes_decrypt_cbc(const AES_key_t *key, size_t len,
-    uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN])
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
+    const uint8_t iv[static AES_BLOCK_LEN])
 {
 	uint8x16_t ivreg, nextiv, tmp;
-	size_t i;
+	uint8_t block[AES_BLOCK_LEN], *from, *to;
+	size_t fromseglen, oseglen, seglen, toseglen;
+
+	KASSERT(len % AES_BLOCK_LEN == 0,
+	    ("%s: length %zu not a multiple of the block size", __func__, len));
 
-	len /= AES_BLOCK_LEN;
 	ivreg = vld1q_u8(iv);
-	for (i = 0; i < len; i++) {
-		nextiv = vld1q_u8(buf);
-		tmp = armv8_aes_dec(key->aes_rounds - 1,
-		    (const void*)key->aes_key, nextiv);
-		vst1q_u8(buf, veorq_u8(tmp, ivreg));
-		ivreg = nextiv;
-		buf += AES_BLOCK_LEN;
+	for (; len > 0; len -= seglen) {
+		from = crypto_cursor_segment(fromc, &fromseglen);
+		to = crypto_cursor_segment(toc, &toseglen);
+
+		seglen = ulmin(len, ulmin(fromseglen, toseglen));
+		if (seglen < AES_BLOCK_LEN) {
+			crypto_cursor_copydata(fromc, AES_BLOCK_LEN, block);
+			nextiv = vld1q_u8(block);
+			tmp = armv8_aes_dec(key->aes_rounds - 1,
+			    (const void *)key->aes_key, nextiv);
+			vst1q_u8(block, veorq_u8(tmp, ivreg));
+			ivreg = nextiv;
+			crypto_cursor_copyback(toc, AES_BLOCK_LEN, block);
+			seglen = AES_BLOCK_LEN;
+		} else {
+			for (oseglen = seglen; seglen >= AES_BLOCK_LEN;
+			    seglen -= AES_BLOCK_LEN) {
+				nextiv = vld1q_u8(from);
+				tmp = armv8_aes_dec(key->aes_rounds - 1,
+				    (const void *)key->aes_key, nextiv);
+				vst1q_u8(to, veorq_u8(tmp, ivreg));
+				ivreg = nextiv;
+				from += AES_BLOCK_LEN;
+				to += AES_BLOCK_LEN;
+			}
+			crypto_cursor_advance(fromc, oseglen - seglen);
+			crypto_cursor_advance(toc, oseglen - seglen);
+			seglen = oseglen - seglen;
+		}
 	}
+
+	explicit_bzero(block, sizeof(block));
 }
 
 #define	AES_XTS_BLOCKSIZE	16
@@ -180,12 +236,18 @@ armv8_aes_crypt_xts_block(int rounds, const uint8x16_t *key_schedule,
 
 static void
 armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule,
-    const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from,
-    uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt)
+    const uint8x16_t *tweak_schedule, size_t len,
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
+    const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt)
 {
 	uint8x16_t tweakreg;
+	uint8_t block[AES_XTS_BLOCKSIZE] __aligned(16);
 	uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16);
-	size_t i, cnt;
+	uint8_t *from, *to;
+	size_t fromseglen, oseglen, seglen, toseglen;
+
+	KASSERT(len % AES_XTS_BLOCKSIZE == 0,
+	    ("%s: length %zu not a multiple of the block size", __func__, len));
 
 	/*
 	 * Prepare tweak as E_k2(IV). IV is specified as LE representation
@@ -201,38 +263,57 @@ armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule,
 	tweakreg = vld1q_u8(tweak);
 	tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg);
 
-	cnt = len / AES_XTS_BLOCKSIZE;
-	for (i = 0; i < cnt; i++) {
-		armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg,
-		    from, to, do_encrypt);
-		from += AES_XTS_BLOCKSIZE;
-		to += AES_XTS_BLOCKSIZE;
+	for (; len > 0; len -= seglen) {
+		from = crypto_cursor_segment(fromc, &fromseglen);
+		to = crypto_cursor_segment(toc, &toseglen);
+
+		seglen = ulmin(len, ulmin(fromseglen, toseglen));
+		if (seglen < AES_XTS_BLOCKSIZE) {
+			printf("%d seglen %zu\n", __LINE__, seglen);
+			crypto_cursor_copydata(fromc, AES_XTS_BLOCKSIZE, block);
+			armv8_aes_crypt_xts_block(rounds, data_schedule,
+			    &tweakreg, block, block, do_encrypt);
+			crypto_cursor_copyback(toc, AES_XTS_BLOCKSIZE, block);
+			seglen = AES_XTS_BLOCKSIZE;
+		} else {
+			printf("%d seglen %zu\n", __LINE__, seglen);
+			for (oseglen = seglen; seglen >= AES_XTS_BLOCKSIZE;
+			    seglen -= AES_XTS_BLOCKSIZE) {
+				armv8_aes_crypt_xts_block(rounds, data_schedule,
+				    &tweakreg, from, to, do_encrypt);
+				from += AES_XTS_BLOCKSIZE;
+				to += AES_XTS_BLOCKSIZE;
+			}
+			seglen = oseglen - seglen;
+			crypto_cursor_advance(fromc, seglen);
+			crypto_cursor_advance(toc, seglen);
+		}
 	}
+
+	explicit_bzero(block, sizeof(block));
 }
 
 void
 armv8_aes_encrypt_xts(AES_key_t *data_schedule,
-    const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
-    const uint8_t iv[static AES_BLOCK_LEN])
+    const void *tweak_schedule, size_t len, struct crypto_buffer_cursor *fromc,
+    struct crypto_buffer_cursor *toc, const uint8_t iv[static AES_BLOCK_LEN])
 {
-
 	armv8_aes_crypt_xts(data_schedule->aes_rounds,
-	    (const void *)&data_schedule->aes_key, tweak_schedule, len, from,
-	    to, iv, 1);
+	    (const void *)&data_schedule->aes_key, tweak_schedule, len, fromc,
+	    toc, iv, 1);
 }
 
 void
 armv8_aes_decrypt_xts(AES_key_t *data_schedule,
-    const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+    const void *tweak_schedule, size_t len,
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
     const uint8_t iv[static AES_BLOCK_LEN])
 {
-
 	armv8_aes_crypt_xts(data_schedule->aes_rounds,
-	    (const void *)&data_schedule->aes_key, tweak_schedule, len, from,
-	    to,iv, 0);
+	    (const void *)&data_schedule->aes_key, tweak_schedule, len, fromc,
+	    toc, iv, 0);
 
 }
-
 #define	AES_INC_COUNTER(counter)				\
 	do {							\
 		for (int pos = AES_BLOCK_LEN - 1;		\
@@ -296,115 +377,161 @@ armv8_aes_gmac_finish(struct armv8_gcm_state *s, size_t len,
 	s->Xi.u[1] ^= s->EK0.u[1];
 }
 
+static void
+armv8_aes_encrypt_gcm_block(struct armv8_gcm_state *s, AES_key_t *aes_key,
+    const uint64_t *from, uint64_t *to)
+{
+	aes_v8_encrypt(s->aes_counter, s->EKi.c, aes_key);
+	AES_INC_COUNTER(s->aes_counter);
+	to[0] = from[0] ^ s->EKi.u[0];
+	to[1] = from[1] ^ s->EKi.u[1];
+}
+
+static void
+armv8_aes_decrypt_gcm_block(struct armv8_gcm_state *s, AES_key_t *aes_key,
+    const uint64_t *from, uint64_t *to)
+{
+	armv8_aes_encrypt_gcm_block(s, aes_key, from, to);
+}
+
 void
 armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
-    const uint8_t *from, uint8_t *to,
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
     size_t authdatalen, const uint8_t *authdata,
     uint8_t tag[static GMAC_DIGEST_LEN],
     const uint8_t iv[static AES_GCM_IV_LEN],
     const __uint128_val_t *Htable)
 {
 	struct armv8_gcm_state s;
-	const uint64_t *from64;
-	uint64_t *to64;
-	uint8_t block[AES_BLOCK_LEN];
-	size_t i, trailer;
+	uint8_t block[AES_BLOCK_LEN] __aligned(AES_BLOCK_LEN);
+	uint64_t *from64, *to64;
+	size_t fromseglen, i, olen, oseglen, seglen, toseglen;
 
 	armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable);
 
-	from64 = (const uint64_t *)from;
-	to64 = (uint64_t *)to;
-	trailer = len % AES_BLOCK_LEN;
-
-	for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
-		aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
-		AES_INC_COUNTER(s.aes_counter);
-		to64[0] = from64[0] ^ s.EKi.u[0];
-		to64[1] = from64[1] ^ s.EKi.u[1];
-		gcm_ghash_v8(s.Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN);
-
-		to64 += 2;
-		from64 += 2;
-	}
-
-	to += (len - trailer);
-	from += (len - trailer);
-
-	if (trailer) {
-		aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
-		AES_INC_COUNTER(s.aes_counter);
-		memset(block, 0, sizeof(block));
-		for (i = 0; i < trailer; i++) {
-			block[i] = to[i] = from[i] ^ s.EKi.c[i];
+	for (olen = len; len > 0; len -= seglen) {
+		from64 = crypto_cursor_segment(fromc, &fromseglen);
+		to64 = crypto_cursor_segment(toc, &toseglen);
+
+		seglen = ulmin(len, ulmin(fromseglen, toseglen));
+		if (seglen < AES_BLOCK_LEN) {
+			seglen = ulmin(len, AES_BLOCK_LEN);
+
+			memset(block, 0, sizeof(block));
+			crypto_cursor_copydata(fromc, (int)seglen, block);
+
+			if (seglen == AES_BLOCK_LEN) {
+				armv8_aes_encrypt_gcm_block(&s, aes_key,
+				    (uint64_t *)block, (uint64_t *)block);
+			} else {
+				aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
+				AES_INC_COUNTER(s.aes_counter);
+				for (i = 0; i < seglen; i++)
+					block[i] ^= s.EKi.c[i];
+			}
+			gcm_ghash_v8(s.Xi.u, Htable, block, seglen);
+
+			crypto_cursor_copyback(toc, (int)seglen, block);
+		} else {
+			for (oseglen = seglen; seglen >= AES_BLOCK_LEN;
+			    seglen -= AES_BLOCK_LEN) {
+				armv8_aes_encrypt_gcm_block(&s, aes_key, from64,
+				    to64);
+				gcm_ghash_v8(s.Xi.u, Htable, (uint8_t *)to64,
+				    AES_BLOCK_LEN);
+
+				from64 += 2;
+				to64 += 2;
+			}
+
+			seglen = oseglen - seglen;
+			crypto_cursor_advance(fromc, seglen);
+			crypto_cursor_advance(toc, seglen);
 		}
-
-		gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
 	}
 
-	armv8_aes_gmac_finish(&s, len, authdatalen, Htable);
+	armv8_aes_gmac_finish(&s, olen, authdatalen, Htable);
 	memcpy(tag, s.Xi.c, GMAC_DIGEST_LEN);
 
+	explicit_bzero(block, sizeof(block));
 	explicit_bzero(&s, sizeof(s));
 }
 
 int
 armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
-    const uint8_t *from, uint8_t *to,
+    struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc,
     size_t authdatalen, const uint8_t *authdata,
     const uint8_t tag[static GMAC_DIGEST_LEN],
     const uint8_t iv[static AES_GCM_IV_LEN],
     const __uint128_val_t *Htable)
 {
 	struct armv8_gcm_state s;
-	const uint64_t *from64;
-	uint64_t *to64;
-	uint8_t block[AES_BLOCK_LEN];
-	size_t i, trailer;
+	struct crypto_buffer_cursor fromcc;
+	uint8_t block[AES_BLOCK_LEN] __aligned(AES_BLOCK_LEN), *from;
+	uint64_t *block64, *from64, *to64;
+	size_t fromseglen, olen, oseglen, seglen, toseglen;
 	int error;
 
-	error = 0;
-
 	armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable);
 
-	trailer = len % AES_BLOCK_LEN;
-	if (len - trailer > 0)
-		gcm_ghash_v8(s.Xi.u, Htable, from, len - trailer);
-	if (trailer > 0) {
-		memset(block, 0, sizeof(block));
-		memcpy(block, from + len - trailer, trailer);
-		gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
+	crypto_cursor_copy(fromc, &fromcc);
+	for (olen = len; len > 0; len -= seglen) {
+		from = crypto_cursor_segment(&fromcc, &fromseglen);
+		seglen = ulmin(len, fromseglen);
+		seglen -= seglen % AES_BLOCK_LEN;
+		if (seglen > 0) {
+			gcm_ghash_v8(s.Xi.u, Htable, from, seglen);
+			crypto_cursor_advance(&fromcc, seglen);
+		} else {
+			memset(block, 0, sizeof(block));
+			seglen = ulmin(len, AES_BLOCK_LEN);
+			crypto_cursor_copydata(&fromcc, seglen, block);
+			gcm_ghash_v8(s.Xi.u, Htable, block, seglen);
+		}
 	}
 
-	armv8_aes_gmac_finish(&s, len, authdatalen, Htable);
+	armv8_aes_gmac_finish(&s, olen, authdatalen, Htable);
 
 	if (timingsafe_bcmp(tag, s.Xi.c, GMAC_DIGEST_LEN) != 0) {
 		error = EBADMSG;
 		goto out;
 	}
 
-	from64 = (const uint64_t *)from;
-	to64 = (uint64_t *)to;
+	block64 = (uint64_t *)block;
+	for (len = olen; len > 0; len -= seglen) {
+		from64 = crypto_cursor_segment(fromc, &fromseglen);
+		to64 = crypto_cursor_segment(toc, &toseglen);
 
-	for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
-		aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
-		AES_INC_COUNTER(s.aes_counter);
-		to64[0] = from64[0] ^ s.EKi.u[0];
-		to64[1] = from64[1] ^ s.EKi.u[1];
-		to64 += 2;
-		from64 += 2;
-	}
+		seglen = ulmin(len, ulmin(fromseglen, toseglen));
+		if (seglen < AES_BLOCK_LEN) {
+			seglen = ulmin(len, AES_BLOCK_LEN);
 
-	to += (len - trailer);
-	from += (len - trailer);
+			memset(block, 0, sizeof(block));
+			crypto_cursor_copydata(fromc, seglen, block);
 
-	if (trailer) {
-		aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
-		AES_INC_COUNTER(s.aes_counter);
-		for (i = 0; i < trailer; i++)
-			to[i] = from[i] ^ s.EKi.c[i];
+			armv8_aes_decrypt_gcm_block(&s, aes_key, block64,
+			    block64);
+
+			crypto_cursor_copyback(toc, (int)seglen, block);
+		} else {
+			for (oseglen = seglen; seglen >= AES_BLOCK_LEN;
+			    seglen -= AES_BLOCK_LEN) {
+				armv8_aes_decrypt_gcm_block(&s, aes_key, from64,
+				    to64);
+
+				from64 += 2;
+				to64 += 2;
+			}
+
+			seglen = oseglen - seglen;
+			crypto_cursor_advance(fromc, seglen);
+			crypto_cursor_advance(toc, seglen);
+		}
 	}
 
+	error = 0;
 out:
+	explicit_bzero(block, sizeof(block));
 	explicit_bzero(&s, sizeof(s));
 	return (error);
 }