cryptosoft: Use multi-block encrypt/decrypt for AES-GCM.

Reviewed by:	markj
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D33756

1 files changed, 91 insertions, 39 deletions

diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c
index 864873944fd4..36081589cc65 100644
--- a/sys/opencrypto/cryptosoft.c
+++ b/sys/opencrypto/cryptosoft.c
@@ -411,12 +411,12 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
 	struct crypto_buffer_cursor cc_in, cc_out;
 	const u_char *inblk;
 	u_char *outblk;
+	size_t inlen, outlen, todo;
 	const struct swcr_auth *swa;
 	const struct swcr_encdec *swe;
 	const struct enc_xform *exf;
 	void *ctx;
 	uint32_t *blkp;
-	size_t len;
 	int blksz, error, ivlen, r, resid;
 
 	swa = &ses->swcr_auth;
@@ -441,30 +441,30 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
 
 	/* Supply MAC with AAD */
 	if (crp->crp_aad != NULL) {
-		len = rounddown(crp->crp_aad_length, blksz);
-		if (len != 0)
-			exf->update(ctx, crp->crp_aad, len);
-		if (crp->crp_aad_length != len) {
+		inlen = rounddown2(crp->crp_aad_length, blksz);
+		if (inlen != 0)
+			exf->update(ctx, crp->crp_aad, inlen);
+		if (crp->crp_aad_length != inlen) {
 			memset(blk, 0, blksz);
-			memcpy(blk, (char *)crp->crp_aad + len,
-			    crp->crp_aad_length - len);
+			memcpy(blk, (char *)crp->crp_aad + inlen,
+			    crp->crp_aad_length - inlen);
 			exf->update(ctx, blk, blksz);
 		}
 	} else {
 		crypto_cursor_init(&cc_in, &crp->crp_buf);
 		crypto_cursor_advance(&cc_in, crp->crp_aad_start);
 		for (resid = crp->crp_aad_length; resid >= blksz;
-		     resid -= len) {
-			inblk = crypto_cursor_segment(&cc_in, &len);
-			if (len >= blksz) {
-				len = rounddown(MIN(len, resid), blksz);
-				crypto_cursor_advance(&cc_in, len);
+		     resid -= inlen) {
+			inblk = crypto_cursor_segment(&cc_in, &inlen);
+			if (inlen >= blksz) {
+				inlen = rounddown2(MIN(inlen, resid), blksz);
+				crypto_cursor_advance(&cc_in, inlen);
 			} else {
-				len = blksz;
-				crypto_cursor_copydata(&cc_in, len, blk);
+				inlen = blksz;
+				crypto_cursor_copydata(&cc_in, inlen, blk);
 				inblk = blk;
 			}
-			exf->update(ctx, inblk, len);
+			exf->update(ctx, inblk, inlen);
 		}
 		if (resid > 0) {
 			memset(blk, 0, blksz);
@@ -476,31 +476,57 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
 	/* Do encryption with MAC */
 	crypto_cursor_init(&cc_in, &crp->crp_buf);
 	crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+	inblk = crypto_cursor_segment(&cc_in, &inlen);
 	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
 		crypto_cursor_init(&cc_out, &crp->crp_obuf);
 		crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
 	} else
 		cc_out = cc_in;
-	for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) {
-		inblk = crypto_cursor_segment(&cc_in, &len);
-		if (len < blksz) {
+	outblk = crypto_cursor_segment(&cc_out, &outlen);
+
+	for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
+		if (inlen < blksz) {
 			crypto_cursor_copydata(&cc_in, blksz, blk);
 			inblk = blk;
-		} else {
-			crypto_cursor_advance(&cc_in, blksz);
+			inlen = blksz;
 		}
+
 		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
-			outblk = crypto_cursor_segment(&cc_out, &len);
-			if (len < blksz)
+			if (outlen < blksz) {
 				outblk = blk;
-			exf->encrypt(ctx, inblk, outblk);
-			exf->update(ctx, outblk, blksz);
-			if (outblk == blk)
+				outlen = blksz;
+			}
+
+			todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
+			    blksz);
+
+			exf->encrypt_multi(ctx, inblk, outblk, todo);
+			exf->update(ctx, outblk, todo);
+
+			if (outblk == blk) {
 				crypto_cursor_copyback(&cc_out, blksz, blk);
-			else
-				crypto_cursor_advance(&cc_out, blksz);
+				outblk = crypto_cursor_segment(&cc_out, &outlen);
+			} else {
+				crypto_cursor_advance(&cc_out, todo);
+				outlen -= todo;
+				outblk += todo;
+				if (outlen == 0)
+					outblk = crypto_cursor_segment(&cc_out,
+					    &outlen);
+			}
 		} else {
-			exf->update(ctx, inblk, blksz);
+			todo = rounddown2(MIN(resid, inlen), blksz);
+			exf->update(ctx, inblk, todo);
+		}
+
+		if (inblk == blk) {
+			inblk = crypto_cursor_segment(&cc_in, &inlen);
+		} else {
+			crypto_cursor_advance(&cc_in, todo);
+			inlen -= todo;
+			inblk += todo;
+			if (inlen == 0)
+				inblk = crypto_cursor_segment(&cc_in, &inlen);
 		}
 	}
 	if (resid > 0) {
@@ -537,22 +563,48 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
 		/* tag matches, decrypt data */
 		crypto_cursor_init(&cc_in, &crp->crp_buf);
 		crypto_cursor_advance(&cc_in, crp->crp_payload_start);
+		inblk = crypto_cursor_segment(&cc_in, &inlen);
+
 		for (resid = crp->crp_payload_length; resid > blksz;
-		     resid -= blksz) {
-			inblk = crypto_cursor_segment(&cc_in, &len);
-			if (len < blksz) {
+		     resid -= todo) {
+			if (inlen < blksz) {
 				crypto_cursor_copydata(&cc_in, blksz, blk);
 				inblk = blk;
-			} else
-				crypto_cursor_advance(&cc_in, blksz);
-			outblk = crypto_cursor_segment(&cc_out, &len);
-			if (len < blksz)
+				inlen = blksz;
+			}
+			if (outlen < blksz) {
 				outblk = blk;
-			exf->decrypt(ctx, inblk, outblk);
-			if (outblk == blk)
+				outlen = blksz;
+			}
+
+			todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
+			    blksz);
+
+			exf->decrypt_multi(ctx, inblk, outblk, todo);
+
+			if (inblk == blk) {
+				inblk = crypto_cursor_segment(&cc_in, &inlen);
+			} else {
+				crypto_cursor_advance(&cc_in, todo);
+				inlen -= todo;
+				inblk += todo;
+				if (inlen == 0)
+					inblk = crypto_cursor_segment(&cc_in,
+					    &inlen);
+			}
+
+			if (outblk == blk) {
 				crypto_cursor_copyback(&cc_out, blksz, blk);
-			else
-				crypto_cursor_advance(&cc_out, blksz);
+				outblk = crypto_cursor_segment(&cc_out,
+				    &outlen);
+			} else {
+				crypto_cursor_advance(&cc_out, todo);
+				outlen -= todo;
+				outblk += todo;
+				if (outlen == 0)
+					outblk = crypto_cursor_segment(&cc_out,
+					    &outlen);
+			}
 		}
 		if (resid > 0) {
 			crypto_cursor_copydata(&cc_in, resid, blk);