armv8crypto: Fix some edge cases in the AES-GCM implementation

- We were only hashing up to the first 16 bytes of the AAD.
- When computing the digest during decryption, handle the case where
  len == trailer, i.e., len < AES_BLOCK_LEN, properly.

While here:

- trailer is always smaller than AES_BLOCK_LEN, so remove a pair of
  unnecessary modulus operations.
- Replace some byte-by-byte loops with memcpy() and memset() calls.
  In particular, zero the full block before copying a partial block into
  it since we do that elsewhere and it means that the memset() length is
  known at compile time.

Reviewed by:	jhb
Sponsored by:	Ampere Computing
Submitted by:	Klara, Inc.
MFC after:	3 days
Differential Revision:	https://reviews.freebsd.org/D28501

1 files changed, 29 insertions, 18 deletions

diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c
index ea93f1b9a176..2f880258bf46 100644
--- a/sys/crypto/armv8/armv8_crypto_wrap.c
+++ b/sys/crypto/armv8/armv8_crypto_wrap.c
@@ -263,9 +263,16 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
 	aes_counter[AES_BLOCK_LEN - 1] = 2;
 
 	memset(Xi.c, 0, sizeof(Xi.c));
-	memset(block, 0, sizeof(block));
-	memcpy(block, authdata, min(authdatalen, sizeof(block)));
-	gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+	trailer = authdatalen % AES_BLOCK_LEN;
+	if (authdatalen - trailer > 0) {
+		gcm_ghash_v8(Xi.u, Htable, authdata, authdatalen - trailer);
+		authdata += authdatalen - trailer;
+	}
+	if (trailer > 0 || authdatalen == 0) {
+		memset(block, 0, sizeof(block));
+		memcpy(block, authdata, trailer);
+		gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+	}
 
 	from64 = (const uint64_t*)from;
 	to64 = (uint64_t*)to;
@@ -288,13 +295,11 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
 	if (trailer) {
 		aes_v8_encrypt(aes_counter, EKi.c, aes_key);
 		AES_INC_COUNTER(aes_counter);
+		memset(block, 0, sizeof(block));
 		for (i = 0; i < trailer; i++) {
-			block[i] = to[i] = from[i] ^ EKi.c[i % AES_BLOCK_LEN];
+			block[i] = to[i] = from[i] ^ EKi.c[i];
 		}
 
-		for (; i < AES_BLOCK_LEN; i++)
-			block[i] = 0;
-
 		gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
 	}
 
@@ -343,17 +348,23 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
 	aes_v8_encrypt(aes_counter, EK0.c, aes_key);
 
 	memset(Xi.c, 0, sizeof(Xi.c));
-	memset(block, 0, sizeof(block));
-	memcpy(block, authdata, min(authdatalen, sizeof(block)));
-	gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
-	trailer = len % AES_BLOCK_LEN;
-	gcm_ghash_v8(Xi.u, Htable, from, len - trailer);
+	trailer = authdatalen % AES_BLOCK_LEN;
+	if (authdatalen - trailer > 0) {
+		gcm_ghash_v8(Xi.u, Htable, authdata, authdatalen - trailer);
+		authdata += authdatalen - trailer;
+	}
+	if (trailer > 0 || authdatalen == 0) {
+		memset(block, 0, sizeof(block));
+		memcpy(block, authdata, trailer);
+		gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
+	}
 
-	if (trailer) {
-		for (i = 0; i < trailer; i++)
-			block[i] = from[len - trailer + i];
-		for (; i < AES_BLOCK_LEN; i++)
-			block[i] = 0;
+	trailer = len % AES_BLOCK_LEN;
+	if (len - trailer > 0)
+		gcm_ghash_v8(Xi.u, Htable, from, len - trailer);
+	if (trailer > 0) {
+		memset(block, 0, sizeof(block));
+		memcpy(block, from + len - trailer, trailer);
 		gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN);
 	}
 
@@ -392,7 +403,7 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
 		aes_v8_encrypt(aes_counter, EKi.c, aes_key);
 		AES_INC_COUNTER(aes_counter);
 		for (i = 0; i < trailer; i++)
-			to[i] = from[i] ^ EKi.c[i % AES_BLOCK_LEN];
+			to[i] = from[i] ^ EKi.c[i];
 	}
 
 out: