ossl: Add support for armv7

OpenSSL provides implementations of several AES modes which use
bitslicing and can be accelerated on CPUs which support the NEON
extension.  This patch adds arm platform support to ossl(4) and provides
an AES-CBC implementation, though bsaes_cbc_encrypt() only implements
decryption.  The real goal is to provide an accelerated AES-GCM
implementation; this will be added in a subsequent patch.

Initially derived from https://reviews.freebsd.org/D37420.

Reviewed by:	jhb
Sponsored by:	Klara, Inc.
Sponsored by:	Stormshield
MFC after:	3 months
Differential Revision:	https://reviews.freebsd.org/D41304

9 files changed, 275 insertions, 6 deletions

diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 772bbb8b0e3b..3643f90b1440 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -135,6 +135,16 @@ libkern/ucmpdi2.c		standard
 libkern/udivdi3.c		standard
 libkern/umoddi3.c		standard
 
+crypto/openssl/ossl_arm.c		optional ossl
+crypto/openssl/arm/aes-armv4.S		optional ossl
+crypto/openssl/arm/bsaes-armv7.S	optional ossl	\
+	compile-with	"${CC} -D__KERNEL__ -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}"
+crypto/openssl/arm/chacha-armv4.S	optional ossl
+crypto/openssl/arm/poly1305-armv4.S	optional ossl
+crypto/openssl/arm/sha1-armv4-large.S	optional ossl
+crypto/openssl/arm/sha256-armv4.S	optional ossl
+crypto/openssl/arm/sha512-armv4.S	optional ossl
+
 # Annapurna support
 arm/annapurna/alpine/alpine_ccu.c		optional al_ccu fdt
 arm/annapurna/alpine/alpine_nb_service.c	optional al_nb_service fdt
diff --git a/sys/crypto/openssl/arm/arm_arch.h b/sys/crypto/openssl/arm/arm_arch.h
new file mode 100644
index 000000000000..8b7105571d78
--- /dev/null
+++ b/sys/crypto/openssl/arm/arm_arch.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef OSSL_CRYPTO_ARM_ARCH_H
+# define OSSL_CRYPTO_ARM_ARCH_H
+
+# if !defined(__ARM_ARCH__)
+#  if defined(__CC_ARM)
+#   define __ARM_ARCH__ __TARGET_ARCH_ARM
+#   if defined(__BIG_ENDIAN)
+#    define __ARMEB__
+#   else
+#    define __ARMEL__
+#   endif
+#  elif defined(__GNUC__)
+#   if   defined(__aarch64__)
+#    define __ARM_ARCH__ 8
+#    if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__
+#     define __ARMEB__
+#    else
+#     define __ARMEL__
+#    endif
+  /*
+   * Why doesn't gcc define __ARM_ARCH__? Instead it defines
+   * bunch of below macros. See all_architectures[] table in
+   * gcc/config/arm/arm.c. On a side note it defines
+   * __ARMEL__/__ARMEB__ for little-/big-endian.
+   */
+#   elif defined(__ARM_ARCH)
+#    define __ARM_ARCH__ __ARM_ARCH
+#   elif defined(__ARM_ARCH_8A__)
+#    define __ARM_ARCH__ 8
+#   elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)     || \
+        defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__)     || \
+        defined(__ARM_ARCH_7EM__)
+#    define __ARM_ARCH__ 7
+#   elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__)     || \
+        defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__)     || \
+        defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__)    || \
+        defined(__ARM_ARCH_6T2__)
+#    define __ARM_ARCH__ 6
+#   elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__)     || \
+        defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__)    || \
+        defined(__ARM_ARCH_5TEJ__)
+#    define __ARM_ARCH__ 5
+#   elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__)
+#    define __ARM_ARCH__ 4
+#   else
+#    error "unsupported ARM architecture"
+#   endif
+#  endif
+# endif
+
+# if !defined(__ARM_MAX_ARCH__)
+#  define __ARM_MAX_ARCH__ __ARM_ARCH__
+# endif
+
+# if __ARM_MAX_ARCH__<__ARM_ARCH__
+#  error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__"
+# elif __ARM_MAX_ARCH__!=__ARM_ARCH__
+#  if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__)
+#   error "can't build universal big-endian binary"
+#  endif
+# endif
+
+# ifndef __ASSEMBLER__
+extern unsigned int OPENSSL_armcap_P;
+# endif
+
+# define ARMV7_NEON      (1<<0)
+# define ARMV7_TICK      (1<<1)
+# define ARMV8_AES       (1<<2)
+# define ARMV8_SHA1      (1<<3)
+# define ARMV8_SHA256    (1<<4)
+# define ARMV8_PMULL     (1<<5)
+# define ARMV8_SHA512    (1<<6)
+
+#endif
diff --git a/sys/crypto/openssl/ossl_aes.c b/sys/crypto/openssl/ossl_aes.c
index 65b6f126736e..b6425035f031 100644
--- a/sys/crypto/openssl/ossl_aes.c
+++ b/sys/crypto/openssl/ossl_aes.c
@@ -39,6 +39,8 @@
 #include <crypto/openssl/ossl_x86.h>
 #elif defined (__aarch64__)
 #include <crypto/openssl/ossl_aarch64.h>
+#elif defined (__arm__)
+#include <crypto/openssl/ossl_arm.h>
 #endif
 
 static ossl_cipher_process_t ossl_aes_cbc;
diff --git a/sys/crypto/openssl/ossl_aes_gcm.h b/sys/crypto/openssl/ossl_aes_gcm.h
index cd0c1e324354..90511318da6c 100644
--- a/sys/crypto/openssl/ossl_aes_gcm.h
+++ b/sys/crypto/openssl/ossl_aes_gcm.h
@@ -28,7 +28,6 @@
 #define	_OSSL_AES_GCM_H_
 
 #include <crypto/openssl/ossl_cipher.h>
-#include <crypto/rijndael/rijndael.h>
 
 struct ossl_gcm_context;
 
@@ -64,10 +63,7 @@ struct ossl_gcm_context {
 		unsigned int mres, ares;
 	} gcm;
 
-	struct {
-		uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
-		int rounds;
-	} aes_ks;
+	struct ossl_aes_keysched aes_ks;
 
 	const struct ossl_aes_gcm_ops *ops;
 };
diff --git a/sys/crypto/openssl/ossl_arm.c b/sys/crypto/openssl/ossl_arm.c
new file mode 100644
index 000000000000..1ec95acd74cd
--- /dev/null
+++ b/sys/crypto/openssl/ossl_arm.c
@@ -0,0 +1,59 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2023 Stormshield
+ * Copyright (c) 2023 Semihalf
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+
+#include <machine/elf.h>
+#include <machine/md_var.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+#include <crypto/openssl/arm/arm_arch.h>
+
+ossl_cipher_setkey_t AES_set_encrypt_key;
+ossl_cipher_setkey_t AES_set_decrypt_key;
+
+unsigned int OPENSSL_armcap_P;
+
+void
+ossl_cpuid(struct ossl_softc *sc)
+{
+	if (elf_hwcap & HWCAP_NEON) {
+		OPENSSL_armcap_P |= ARMV7_NEON;
+
+		sc->has_aes = true;
+		ossl_cipher_aes_cbc.set_encrypt_key = AES_set_encrypt_key;
+		ossl_cipher_aes_cbc.set_decrypt_key = AES_set_decrypt_key;
+	}
+}
diff --git a/sys/crypto/openssl/ossl_arm.h b/sys/crypto/openssl/ossl_arm.h
new file mode 100644
index 000000000000..56772feb43f2
--- /dev/null
+++ b/sys/crypto/openssl/ossl_arm.h
@@ -0,0 +1,94 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2023 Stormshield
+ * Copyright (c) 2023 Semihalf
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#ifndef __OSSL_ARM__
+#define __OSSL_ARM__
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+
+#include <opencrypto/cryptodev.h>
+
+struct bsaes_key {
+	struct ossl_aes_keysched ks;
+	int converted;
+#define	BSAES_KEY_SIZE	(128 * (RIJNDAEL_MAXNR - 1) + 2 * AES_BLOCK_LEN)
+	uint8_t bitslice[BSAES_KEY_SIZE] __aligned(8);
+} __aligned(8);
+
+ossl_cipher_encrypt_t ossl_bsaes_cbc_encrypt;
+
+void AES_encrypt(const void *, void *, const void *);
+
+static inline void
+AES_CBC_ENCRYPT(const unsigned char *in, unsigned char *out,
+    size_t length, const void *key, unsigned char *iv, int encrypt)
+{
+	struct bsaes_key bsks;
+	uint32_t iv32[4], scratch[4];
+
+	/*
+	 * bsaes_cbc_encrypt has some special requirements w.r.t input data.
+	 * The key buffer, that normally holds round keys is used as a scratch
+	 * space. 128 bytes per round of extra space is required.
+	 * Another thing is that only decryption is supported.
+	 * In the case of encryption block chaining has to be done in C.
+	 */
+	if (!encrypt) {
+		memcpy(&bsks.ks, key, sizeof(bsks.ks));
+		bsks.converted = 0;
+		ossl_bsaes_cbc_encrypt(in, out, length, &bsks, iv, false);
+		return;
+	}
+
+	length /= AES_BLOCK_LEN;
+	memcpy(iv32, iv, AES_BLOCK_LEN);
+
+	while (length-- > 0) {
+		memcpy(scratch, in, AES_BLOCK_LEN);
+
+		/* XOR plaintext with IV. */
+		scratch[0] ^= iv32[0];
+		scratch[1] ^= iv32[1];
+		scratch[2] ^= iv32[2];
+		scratch[3] ^= iv32[3];
+
+		AES_encrypt(scratch, out, key);
+
+		memcpy(iv32, out, AES_BLOCK_LEN);
+		in += AES_BLOCK_LEN;
+		out += AES_BLOCK_LEN;
+	}
+
+	memcpy(iv, iv32, AES_BLOCK_LEN);
+}
+
+#endif /* __OSSL_ARM__ */
diff --git a/sys/crypto/openssl/ossl_cipher.h b/sys/crypto/openssl/ossl_cipher.h
index 6599524a707f..886fd9f83350 100644
--- a/sys/crypto/openssl/ossl_cipher.h
+++ b/sys/crypto/openssl/ossl_cipher.h
@@ -28,6 +28,9 @@
 #ifndef __OSSL_CIPHER_H__
 #define __OSSL_CIPHER_H__
 
+#include <sys/types.h>
+#include <crypto/rijndael/rijndael.h>
+
 struct ossl_session_cipher;
 struct cryptop;
 struct crypto_session_params;
@@ -50,4 +53,9 @@ struct ossl_cipher {
 	ossl_cipher_process_t	*process;
 };
 
+struct ossl_aes_keysched {
+	uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
+	int rounds;
+};
+
 #endif
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index f9079498dc1f..112a2f345c94 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -565,11 +565,15 @@ _iwlwifi=	iwlwifi
 .if ${MK_SOURCELESS_UCODE} != "no"
 _iwlwififw=	iwlwififw
 .endif
-_ossl=		ossl
 _rtw88=		rtw88
 _vmware=	vmware
 .endif
 
+.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" || \
+    ${MACHINE_CPUARCH} == "i386" || ${MACHINE_ARCH} == "armv7"
+_ossl=		ossl
+.endif
+
 # MAC framework
 .if ${KERN_OPTS:MMAC} || defined(ALL_MODULES)
 _mac_biba=	mac_biba
diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile
index 7f70f19db76c..d26aabf7bff2 100644
--- a/sys/modules/ossl/Makefile
+++ b/sys/modules/ossl/Makefile
@@ -16,6 +16,16 @@ SRCS=	bus_if.h \
 	ossl_sha512.c \
 	${SRCS.${MACHINE_CPUARCH}}
 
+SRCS.arm= \
+	aes-armv4.S \
+	bsaes-armv7.S \
+	chacha-armv4.S \
+	poly1305-armv4.S  \
+	sha1-armv4-large.S \
+	sha256-armv4.S \
+	sha512-armv4.S \
+	ossl_arm.c
+
 SRCS.aarch64= \
 	chacha-armv8.S \
 	poly1305-armv8.S \
@@ -47,6 +57,8 @@ SRCS.i386= \
 	sha512-586.S \
 	ossl_x86.c
 
+CFLAGS.bsaes-armv7.S+=	-D__KERNEL__
+
 # For arm64, we are forced to rewrite the compiler invocation for the assembly
 # files, to remove -mgeneral-regs-only.
 ${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S