Add a kernel crypto driver using assembly routines from OpenSSL.

Currently, this supports SHA1 and SHA2-{224,256,384,512} both as plain
hashes and in HMAC mode on both amd64 and i386.  It uses the SHA
intrinsics when present similar to aesni(4), but uses SSE/AVX
instructions when they are not.

Note that some files from OpenSSL that normally wrap the assembly
routines have been adapted to export methods usable by 'struct
auth_xform' as is used by existing software crypto routines.

Reviewed by:	gallatin, jkim, delphij, gnn
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D26821

7 files changed, 1090 insertions, 0 deletions

diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c
new file mode 100644
index 000000000000..f4098dde2ce5
--- /dev/null
+++ b/sys/crypto/openssl/ossl.c
@@ -0,0 +1,369 @@
+/*
+ * Copyright (c) 2020 Netflix, Inc
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+/*
+ * A driver for the OpenCrypto framework which uses assembly routines
+ * from OpenSSL.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <machine/fpu.h>
+#include <machine/md_var.h>
+#include <x86/cputypes.h>
+#include <x86/specialreg.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include <crypto/openssl/ossl.h>
+
+#include "cryptodev_if.h"
+
+struct ossl_softc {
+	int32_t sc_cid;
+};
+
+struct ossl_session_hash {
+	struct ossl_hash_context ictx;
+	struct ossl_hash_context octx;
+	struct auth_hash *axf;
+	u_int mlen;
+};
+
+struct ossl_session {
+	struct ossl_session_hash hash;
+};
+
+/*
+ * See OPENSSL_ia32cap(3).
+ *
+ * [0] = cpu_feature but with a few custom bits
+ * [1] = cpu_feature2 but with AMD XOP in bit 11
+ * [2] = cpu_stdext_feature
+ * [3] = 0
+ */
+unsigned int OPENSSL_ia32cap_P[4];
+
+static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto");
+
+static void
+ossl_cpuid(void)
+{
+	uint64_t xcr0;
+	u_int regs[4];
+	u_int max_cores;
+
+	/* Derived from OpenSSL_ia32_cpuid. */
+
+	OPENSSL_ia32cap_P[0] = cpu_feature & ~(CPUID_B20 | CPUID_IA64);
+	if (cpu_vendor_id == CPU_VENDOR_INTEL) {
+		OPENSSL_ia32cap_P[0] |= CPUID_IA64;
+		if ((cpu_id & 0xf00) != 0xf00)
+			OPENSSL_ia32cap_P[0] |= CPUID_B20;
+	}
+
+	/* Only leave CPUID_HTT on if HTT is present. */
+	if (cpu_vendor_id == CPU_VENDOR_AMD && cpu_exthigh >= 0x80000008) {
+		max_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1;
+		if (cpu_feature & CPUID_HTT) {
+			if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 <= max_cores)
+				OPENSSL_ia32cap_P[0] &= ~CPUID_HTT;
+		}
+	} else {
+		if (cpu_high >= 4) {
+			cpuid_count(4, 0, regs);
+			max_cores = (regs[0] >> 26) & 0xfff;
+		} else
+			max_cores = -1;
+	}
+	if (max_cores == 0)
+		OPENSSL_ia32cap_P[0] &= ~CPUID_HTT;
+	else if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 == 0)
+		OPENSSL_ia32cap_P[0] &= ~CPUID_HTT;
+
+	OPENSSL_ia32cap_P[1] = cpu_feature2 & ~AMDID2_XOP;
+	if (cpu_vendor_id == CPU_VENDOR_AMD)
+		OPENSSL_ia32cap_P[1] |= amd_feature2 & AMDID2_XOP;
+
+	OPENSSL_ia32cap_P[2] = cpu_stdext_feature;
+	if ((OPENSSL_ia32cap_P[1] & CPUID2_XSAVE) == 0)
+		OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F |
+		    CPUID_STDEXT_AVX512DQ);
+
+	/* Disable AVX512F on Skylake-X. */
+	if ((cpu_id & 0x0fff0ff0) == 0x00050650)
+		OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F);
+
+	if (cpu_feature2 & CPUID2_OSXSAVE)
+		xcr0 = rxcr(0);
+	else
+		xcr0 = 0;
+
+	if ((xcr0 & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
+	    (XFEATURE_AVX512 | XFEATURE_AVX))
+		OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512VL |
+		    CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512IFMA |
+		    CPUID_STDEXT_AVX512F);
+	if ((xcr0 & XFEATURE_AVX) != XFEATURE_AVX) {
+		OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA);
+		OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2;
+	}
+}
+
+static void
+ossl_identify(driver_t *driver, device_t parent)
+{
+
+	if (device_find_child(parent, "ossl", -1) == NULL)
+		BUS_ADD_CHILD(parent, 10, "ossl", -1);
+}
+
+static int
+ossl_probe(device_t dev)
+{
+
+	device_set_desc(dev, "OpenSSL crypto");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+ossl_attach(device_t dev)
+{
+	struct ossl_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	ossl_cpuid();
+	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session),
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
+	    CRYPTOCAP_F_ACCEL_SOFTWARE);
+	if (sc->sc_cid < 0) {
+		device_printf(dev, "failed to allocate crypto driver id\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static int
+ossl_detach(device_t dev)
+{
+	struct ossl_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	crypto_unregister_all(sc->sc_cid);
+
+	return (0);
+}
+
+static struct auth_hash *
+ossl_lookup_hash(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA1_HMAC:
+		return (&ossl_hash_sha1);
+	case CRYPTO_SHA2_224:
+	case CRYPTO_SHA2_224_HMAC:
+		return (&ossl_hash_sha224);
+	case CRYPTO_SHA2_256:
+	case CRYPTO_SHA2_256_HMAC:
+		return (&ossl_hash_sha256);
+	case CRYPTO_SHA2_384:
+	case CRYPTO_SHA2_384_HMAC:
+		return (&ossl_hash_sha384);
+	case CRYPTO_SHA2_512:
+	case CRYPTO_SHA2_512_HMAC:
+		return (&ossl_hash_sha512);
+	default:
+		return (NULL);
+	}
+}
+
+static int
+ossl_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+
+	if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
+	    0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (ossl_lookup_hash(csp) == NULL)
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
+
+static void
+ossl_setkey_hmac(struct ossl_session *s, const void *key, int klen)
+{
+
+	hmac_init_ipad(s->hash.axf, key, klen, &s->hash.ictx);
+	hmac_init_opad(s->hash.axf, key, klen, &s->hash.octx);
+}
+
+static int
+ossl_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct ossl_session *s;
+	struct auth_hash *axf;
+
+	s = crypto_get_driver_session(cses);
+
+	axf = ossl_lookup_hash(csp);
+	s->hash.axf = axf;
+	if (csp->csp_auth_mlen == 0)
+		s->hash.mlen = axf->hashsize;
+	else
+		s->hash.mlen = csp->csp_auth_mlen;
+
+	if (csp->csp_auth_klen == 0) {
+		axf->Init(&s->hash.ictx);
+	} else {
+		if (csp->csp_auth_key != NULL) {
+			fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
+			ossl_setkey_hmac(s, csp->csp_auth_key,
+			    csp->csp_auth_klen);
+			fpu_kern_leave(curthread, NULL);
+		}
+	}
+	return (0);
+}
+
+static int
+ossl_process(device_t dev, struct cryptop *crp, int hint)
+{
+	struct ossl_hash_context ctx;
+	char digest[HASH_MAX_LEN];
+	const struct crypto_session_params *csp;
+	struct ossl_session *s;
+	struct auth_hash *axf;
+	int error;
+	bool fpu_entered;
+
+	s = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
+	axf = s->hash.axf;
+
+	if (is_fpu_kern_thread(0)) {
+		fpu_entered = false;
+	} else {
+		fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
+		fpu_entered = true;
+	}
+
+	if (crp->crp_auth_key != NULL)
+		ossl_setkey_hmac(s, crp->crp_auth_key, csp->csp_auth_klen);
+
+	ctx = s->hash.ictx;
+
+	if (crp->crp_aad != NULL)
+		error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length);
+	else
+		error = crypto_apply(crp, crp->crp_aad_start,
+		    crp->crp_aad_length, axf->Update, &ctx);
+	if (error)
+		goto out;
+
+	error = crypto_apply(crp, crp->crp_payload_start,
+	    crp->crp_payload_length, axf->Update, &ctx);
+	if (error)
+		goto out;
+
+	axf->Final(digest, &ctx);
+
+	if (csp->csp_auth_klen != 0) {
+		ctx = s->hash.octx;
+		axf->Update(&ctx, digest, axf->hashsize);
+		axf->Final(digest, &ctx);
+	}
+
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		char digest2[HASH_MAX_LEN];
+
+		crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen,
+		    digest2);
+		if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0)
+			error = EBADMSG;
+		explicit_bzero(digest2, sizeof(digest2));
+	} else {
+		crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen,
+		    digest);
+	}
+	explicit_bzero(digest, sizeof(digest));
+
+out:
+	if (fpu_entered)
+		fpu_kern_leave(curthread, NULL);
+
+	crp->crp_etype = error;
+	crypto_done(crp);
+
+	explicit_bzero(&ctx, sizeof(ctx));
+	return (0);
+}
+
+static device_method_t ossl_methods[] = {
+	DEVMETHOD(device_identify,	ossl_identify),
+	DEVMETHOD(device_probe,		ossl_probe),
+	DEVMETHOD(device_attach,	ossl_attach),
+	DEVMETHOD(device_detach,	ossl_detach),
+
+	DEVMETHOD(cryptodev_probesession, ossl_probesession),
+	DEVMETHOD(cryptodev_newsession,	ossl_newsession),
+	DEVMETHOD(cryptodev_process,	ossl_process),
+
+	DEVMETHOD_END
+};
+
+static driver_t ossl_driver = {
+	"ossl",
+	ossl_methods,
+	sizeof(struct ossl_softc)
+};
+
+static devclass_t ossl_devclass;
+
+DRIVER_MODULE(ossl, nexus, ossl_driver, ossl_devclass, NULL, NULL);
+MODULE_VERSION(ossl, 1);
+MODULE_DEPEND(ossl, crypto, 1, 1, 1);
diff --git a/sys/crypto/openssl/ossl.h b/sys/crypto/openssl/ossl.h
new file mode 100644
index 000000000000..5a2d0c9351aa
--- /dev/null
+++ b/sys/crypto/openssl/ossl.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2020 Netflix, Inc
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __OSSL_H__
+#define	__OSSL_H__
+
+/* Compatibility shims. */
+#define	OPENSSL_cleanse		explicit_bzero
+
+/* Used by assembly routines to select CPU-specific variants. */
+extern unsigned int OPENSSL_ia32cap_P[4];
+
+/* Needs to be big enough to hold any hash context. */
+struct ossl_hash_context {
+	uint32_t	dummy[54];
+} __aligned(32);
+
+extern struct auth_hash ossl_hash_sha1;
+extern struct auth_hash ossl_hash_sha224;
+extern struct auth_hash ossl_hash_sha256;
+extern struct auth_hash ossl_hash_sha384;
+extern struct auth_hash ossl_hash_sha512;
+
+#endif /* !__OSSL_H__ */
diff --git a/sys/crypto/openssl/ossl_hash.h b/sys/crypto/openssl/ossl_hash.h
new file mode 100644
index 000000000000..9df8bd526346
--- /dev/null
+++ b/sys/crypto/openssl/ossl_hash.h
@@ -0,0 +1,146 @@
+/*
+ * Copyright 1999-2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * Derived from include/crypto/md32_common.h
+ *
+ * HASH_UPDATE and HASH_FINAL have been updated to work with the
+ * auth_hash interface.
+ */
+
+#if defined(DATA_ORDER_IS_BIG_ENDIAN)
+
+# define HOST_c2l(c,l)  (l =(((unsigned long)(*((c)++)))<<24),          \
+                         l|=(((unsigned long)(*((c)++)))<<16),          \
+                         l|=(((unsigned long)(*((c)++)))<< 8),          \
+                         l|=(((unsigned long)(*((c)++)))    )           )
+# define HOST_l2c(l,c)  (*((c)++)=(unsigned char)(((l)>>24)&0xff),      \
+                         *((c)++)=(unsigned char)(((l)>>16)&0xff),      \
+                         *((c)++)=(unsigned char)(((l)>> 8)&0xff),      \
+                         *((c)++)=(unsigned char)(((l)    )&0xff),      \
+                         l)
+
+#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN)
+
+# define HOST_c2l(c,l)  (l =(((unsigned long)(*((c)++)))    ),          \
+                         l|=(((unsigned long)(*((c)++)))<< 8),          \
+                         l|=(((unsigned long)(*((c)++)))<<16),          \
+                         l|=(((unsigned long)(*((c)++)))<<24)           )
+# define HOST_l2c(l,c)  (*((c)++)=(unsigned char)(((l)    )&0xff),      \
+                         *((c)++)=(unsigned char)(((l)>> 8)&0xff),      \
+                         *((c)++)=(unsigned char)(((l)>>16)&0xff),      \
+                         *((c)++)=(unsigned char)(((l)>>24)&0xff),      \
+                         l)
+
+#endif
+
+/*
+ * Time for some action :-)
+ */
+
+static int
+HASH_UPDATE(void *c_, const void *data_, unsigned int len)
+{
+    HASH_CTX *c = c_;
+    const unsigned char *data = data_;
+    unsigned char *p;
+    HASH_LONG l;
+    size_t n;
+
+    if (len == 0)
+        return 0;
+
+    l = (c->Nl + (((HASH_LONG) len) << 3)) & 0xffffffffUL;
+    if (l < c->Nl)              /* overflow */
+        c->Nh++;
+    c->Nh += (HASH_LONG) (len >> 29); /* might cause compiler warning on
+                                       * 16-bit */
+    c->Nl = l;
+
+    n = c->num;
+    if (n != 0) {
+        p = (unsigned char *)c->data;
+
+        if (len >= HASH_CBLOCK || len + n >= HASH_CBLOCK) {
+            memcpy(p + n, data, HASH_CBLOCK - n);
+            HASH_BLOCK_DATA_ORDER(c, p, 1);
+            n = HASH_CBLOCK - n;
+            data += n;
+            len -= n;
+            c->num = 0;
+            /*
+             * We use memset rather than OPENSSL_cleanse() here deliberately.
+             * Using OPENSSL_cleanse() here could be a performance issue. It
+             * will get properly cleansed on finalisation so this isn't a
+             * security problem.
+             */
+            memset(p, 0, HASH_CBLOCK); /* keep it zeroed */
+        } else {
+            memcpy(p + n, data, len);
+            c->num += (unsigned int)len;
+            return 0;
+        }
+    }
+
+    n = len / HASH_CBLOCK;
+    if (n > 0) {
+        HASH_BLOCK_DATA_ORDER(c, data, n);
+        n *= HASH_CBLOCK;
+        data += n;
+        len -= n;
+    }
+
+    if (len != 0) {
+        p = (unsigned char *)c->data;
+        c->num = (unsigned int)len;
+        memcpy(p, data, len);
+    }
+    return 0;
+}
+
+static void
+HASH_FINAL(uint8_t *md, void *c_)
+{
+    HASH_CTX *c = c_;
+    unsigned char *p = (unsigned char *)c->data;
+    size_t n = c->num;
+
+    p[n] = 0x80;                /* there is always room for one */
+    n++;
+
+    if (n > (HASH_CBLOCK - 8)) {
+        memset(p + n, 0, HASH_CBLOCK - n);
+        n = 0;
+        HASH_BLOCK_DATA_ORDER(c, p, 1);
+    }
+    memset(p + n, 0, HASH_CBLOCK - 8 - n);
+
+    p += HASH_CBLOCK - 8;
+#if   defined(DATA_ORDER_IS_BIG_ENDIAN)
+    (void)HOST_l2c(c->Nh, p);
+    (void)HOST_l2c(c->Nl, p);
+#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN)
+    (void)HOST_l2c(c->Nl, p);
+    (void)HOST_l2c(c->Nh, p);
+#endif
+    p -= HASH_CBLOCK;
+    HASH_BLOCK_DATA_ORDER(c, p, 1);
+    c->num = 0;
+    OPENSSL_cleanse(p, HASH_CBLOCK);
+
+#ifndef HASH_MAKE_STRING
+# error "HASH_MAKE_STRING must be defined!"
+#else
+    HASH_MAKE_STRING(c, md);
+#endif
+
+    return;
+}
diff --git a/sys/crypto/openssl/ossl_sha.h b/sys/crypto/openssl/ossl_sha.h
new file mode 100644
index 000000000000..589ae7638fce
--- /dev/null
+++ b/sys/crypto/openssl/ossl_sha.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __OSSL_SHA_H__
+#define	__OSSL_SHA_H__
+
+/*
+ * This is always included last which permits the namespace hacks below
+ * to work.
+ */
+#define	SHA256_CTX	OSSL_SHA256_CTX
+#define	SHA512_CTX	OSSL_SHA512_CTX
+
+/* From include/openssl/sha.h */
+# define SHA_LONG unsigned int
+
+# define SHA_LBLOCK      16
+# define SHA_CBLOCK      (SHA_LBLOCK*4)/* SHA treats input data as a
+                                        * contiguous array of 32 bit wide
+                                        * big-endian values. */
+
+typedef struct SHAstate_st {
+    SHA_LONG h0, h1, h2, h3, h4;
+    SHA_LONG Nl, Nh;
+    SHA_LONG data[SHA_LBLOCK];
+    unsigned int num;
+} SHA_CTX;
+
+# define SHA256_CBLOCK   (SHA_LBLOCK*4)/* SHA-256 treats input data as a
+                                        * contiguous array of 32 bit wide
+                                        * big-endian values. */
+
+typedef struct SHA256state_st {
+    SHA_LONG h[8];
+    SHA_LONG Nl, Nh;
+    SHA_LONG data[SHA_LBLOCK];
+    unsigned int num, md_len;
+} SHA256_CTX;
+
+/*
+ * SHA-512 treats input data as a
+ * contiguous array of 64 bit
+ * wide big-endian values.
+ */
+# define SHA512_CBLOCK   (SHA_LBLOCK*8)
+
+#  define SHA_LONG64 unsigned long long
+#  define U64(C)     C##ULL
+
+typedef struct SHA512state_st {
+    SHA_LONG64 h[8];
+    SHA_LONG64 Nl, Nh;
+    union {
+        SHA_LONG64 d[SHA_LBLOCK];
+        unsigned char p[SHA512_CBLOCK];
+    } u;
+    unsigned int num, md_len;
+} SHA512_CTX;
+
+#endif /* !__OSSL_SHA_H__ */
diff --git a/sys/crypto/openssl/ossl_sha1.c b/sys/crypto/openssl/ossl_sha1.c
new file mode 100644
index 000000000000..7d9c9938b1bd
--- /dev/null
+++ b/sys/crypto/openssl/ossl_sha1.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_sha.h>
+
+/* sha1-x86_64.S */
+void sha1_block_data_order(SHA_CTX *c, const void *p, size_t len);
+
+/* From crypto/sha/sha_local.h */
+#define DATA_ORDER_IS_BIG_ENDIAN
+
+#define HASH_LONG               SHA_LONG
+#define HASH_CTX                SHA_CTX
+#define HASH_CBLOCK             SHA_CBLOCK
+#define HASH_MAKE_STRING(c,s)   do {    \
+        unsigned long ll;               \
+        ll=(c)->h0; (void)HOST_l2c(ll,(s));     \
+        ll=(c)->h1; (void)HOST_l2c(ll,(s));     \
+        ll=(c)->h2; (void)HOST_l2c(ll,(s));     \
+        ll=(c)->h3; (void)HOST_l2c(ll,(s));     \
+        ll=(c)->h4; (void)HOST_l2c(ll,(s));     \
+        } while (0)
+
+#define HASH_UPDATE                     ossl_sha1_update
+#define HASH_FINAL                      ossl_sha1_final
+#define HASH_INIT                       ossl_sha1_init
+#define HASH_BLOCK_DATA_ORDER           sha1_block_data_order
+
+#define INIT_DATA_h0 0x67452301UL
+#define INIT_DATA_h1 0xefcdab89UL
+#define INIT_DATA_h2 0x98badcfeUL
+#define INIT_DATA_h3 0x10325476UL
+#define INIT_DATA_h4 0xc3d2e1f0UL
+
+static void
+HASH_INIT(void *c_)
+{
+    SHA_CTX *c = c_;
+    memset(c, 0, sizeof(*c));
+    c->h0 = INIT_DATA_h0;
+    c->h1 = INIT_DATA_h1;
+    c->h2 = INIT_DATA_h2;
+    c->h3 = INIT_DATA_h3;
+    c->h4 = INIT_DATA_h4;
+}
+
+#include "ossl_hash.h"
+
+struct auth_hash ossl_hash_sha1 = {
+	.type = CRYPTO_SHA1,
+	.name = "OpenSSL-SHA1",
+	.hashsize = SHA1_HASH_LEN,
+	.ctxsize = sizeof(SHA_CTX),
+	.blocksize = SHA1_BLOCK_LEN,
+	.Init = HASH_INIT,
+	.Update = HASH_UPDATE,
+	.Final = HASH_FINAL,
+};
+
+_Static_assert(sizeof(SHA_CTX) <= sizeof(struct ossl_hash_context),
+    "ossl_hash_context too small");
diff --git a/sys/crypto/openssl/ossl_sha256.c b/sys/crypto/openssl/ossl_sha256.c
new file mode 100644
index 000000000000..8c273bd137fd
--- /dev/null
+++ b/sys/crypto/openssl/ossl_sha256.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2004-2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_sha.h>
+
+/* sha256-x86_64.S */
+void sha256_block_data_order(SHA256_CTX *c, const void *in, size_t num);
+
+/* From crypto/sha/sha256.c */
+
+static void
+ossl_sha224_init(void *c_)
+{
+    SHA256_CTX *c = c_;
+    memset(c, 0, sizeof(*c));
+    c->h[0] = 0xc1059ed8UL;
+    c->h[1] = 0x367cd507UL;
+    c->h[2] = 0x3070dd17UL;
+    c->h[3] = 0xf70e5939UL;
+    c->h[4] = 0xffc00b31UL;
+    c->h[5] = 0x68581511UL;
+    c->h[6] = 0x64f98fa7UL;
+    c->h[7] = 0xbefa4fa4UL;
+    c->md_len = SHA224_DIGEST_LENGTH;
+}
+
+static void
+ossl_sha256_init(void *c_)
+{
+    SHA256_CTX *c = c_;
+    memset(c, 0, sizeof(*c));
+    c->h[0] = 0x6a09e667UL;
+    c->h[1] = 0xbb67ae85UL;
+    c->h[2] = 0x3c6ef372UL;
+    c->h[3] = 0xa54ff53aUL;
+    c->h[4] = 0x510e527fUL;
+    c->h[5] = 0x9b05688cUL;
+    c->h[6] = 0x1f83d9abUL;
+    c->h[7] = 0x5be0cd19UL;
+    c->md_len = SHA256_DIGEST_LENGTH;
+}
+
+
+#define DATA_ORDER_IS_BIG_ENDIAN
+
+#define HASH_LONG               SHA_LONG
+#define HASH_CTX                SHA256_CTX
+#define HASH_CBLOCK             SHA_CBLOCK
+
+/*
+ * Note that FIPS180-2 discusses "Truncation of the Hash Function Output."
+ * default: case below covers for it. It's not clear however if it's
+ * permitted to truncate to amount of bytes not divisible by 4. I bet not,
+ * but if it is, then default: case shall be extended. For reference.
+ * Idea behind separate cases for pre-defined lengths is to let the
+ * compiler decide if it's appropriate to unroll small loops.
+ */
+#define HASH_MAKE_STRING(c,s)   do {    \
+        unsigned long ll;               \
+        unsigned int  nn;               \
+        switch ((c)->md_len)            \
+        {   case SHA224_DIGEST_LENGTH:  \
+                for (nn=0;nn<SHA224_DIGEST_LENGTH/4;nn++)       \
+                {   ll=(c)->h[nn]; (void)HOST_l2c(ll,(s));   }  \
+                break;                  \
+            case SHA256_DIGEST_LENGTH:  \
+                for (nn=0;nn<SHA256_DIGEST_LENGTH/4;nn++)       \
+                {   ll=(c)->h[nn]; (void)HOST_l2c(ll,(s));   }  \
+                break;                  \
+            default:                    \
+                __assert_unreachable(); \
+                break;                  \
+        }                               \
+        } while (0)
+
+#define HASH_UPDATE             ossl_sha256_update
+#define HASH_FINAL              ossl_sha256_final
+#define HASH_BLOCK_DATA_ORDER   sha256_block_data_order
+
+#include "ossl_hash.h"
+
+struct auth_hash ossl_hash_sha224 = {
+	.type = CRYPTO_SHA2_224,
+	.name = "OpenSSL-SHA2-224",
+	.hashsize = SHA2_224_HASH_LEN,
+	.ctxsize = sizeof(SHA256_CTX),
+	.blocksize = SHA2_224_BLOCK_LEN,
+	.Init = ossl_sha224_init,
+	.Update = HASH_UPDATE,
+	.Final = HASH_FINAL,
+};
+
+struct auth_hash ossl_hash_sha256 = {
+	.type = CRYPTO_SHA2_256,
+	.name = "OpenSSL-SHA2-256",
+	.hashsize = SHA2_256_HASH_LEN,
+	.ctxsize = sizeof(SHA256_CTX),
+	.blocksize = SHA2_256_BLOCK_LEN,
+	.Init = ossl_sha256_init,
+	.Update = HASH_UPDATE,
+	.Final = HASH_FINAL,
+};
+
+_Static_assert(sizeof(SHA256_CTX) <= sizeof(struct ossl_hash_context),
+    "ossl_hash_context too small");
diff --git a/sys/crypto/openssl/ossl_sha512.c b/sys/crypto/openssl/ossl_sha512.c
new file mode 100644
index 000000000000..1955feb93e81
--- /dev/null
+++ b/sys/crypto/openssl/ossl_sha512.c
@@ -0,0 +1,258 @@
+/*
+ * Copyright 2004-2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_sha.h>
+
+/* sha512-x86_64.S */
+void sha512_block_data_order(SHA512_CTX *c, const void *in, size_t num);
+
+/* From crypto/sha/sha512.c */
+
+#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
+# define SHA512_BLOCK_CAN_MANAGE_UNALIGNED_DATA
+#endif
+
+static void
+ossl_sha384_init(void *c_)
+{
+    SHA512_CTX *c = c_;
+    c->h[0] = U64(0xcbbb9d5dc1059ed8);
+    c->h[1] = U64(0x629a292a367cd507);
+    c->h[2] = U64(0x9159015a3070dd17);
+    c->h[3] = U64(0x152fecd8f70e5939);
+    c->h[4] = U64(0x67332667ffc00b31);
+    c->h[5] = U64(0x8eb44a8768581511);
+    c->h[6] = U64(0xdb0c2e0d64f98fa7);
+    c->h[7] = U64(0x47b5481dbefa4fa4);
+
+    c->Nl = 0;
+    c->Nh = 0;
+    c->num = 0;
+    c->md_len = SHA384_DIGEST_LENGTH;
+}
+
+static void
+ossl_sha512_init(void *c_)
+{
+    SHA512_CTX *c = c_;
+    c->h[0] = U64(0x6a09e667f3bcc908);
+    c->h[1] = U64(0xbb67ae8584caa73b);
+    c->h[2] = U64(0x3c6ef372fe94f82b);
+    c->h[3] = U64(0xa54ff53a5f1d36f1);
+    c->h[4] = U64(0x510e527fade682d1);
+    c->h[5] = U64(0x9b05688c2b3e6c1f);
+    c->h[6] = U64(0x1f83d9abfb41bd6b);
+    c->h[7] = U64(0x5be0cd19137e2179);
+
+    c->Nl = 0;
+    c->Nh = 0;
+    c->num = 0;
+    c->md_len = SHA512_DIGEST_LENGTH;
+}
+
+static void
+ossl_sha512_final(uint8_t *md, void *c_)
+{
+    SHA512_CTX *c = c_;
+    unsigned char *p = (unsigned char *)c->u.p;
+    size_t n = c->num;
+
+    p[n] = 0x80;                /* There always is a room for one */
+    n++;
+    if (n > (sizeof(c->u) - 16)) {
+        memset(p + n, 0, sizeof(c->u) - n);
+        n = 0;
+        sha512_block_data_order(c, p, 1);
+    }
+
+    memset(p + n, 0, sizeof(c->u) - 16 - n);
+#if _BYTE_ORDER == _BIG_ENDIAN
+    c->u.d[SHA_LBLOCK - 2] = c->Nh;
+    c->u.d[SHA_LBLOCK - 1] = c->Nl;
+#else
+    p[sizeof(c->u) - 1] = (unsigned char)(c->Nl);
+    p[sizeof(c->u) - 2] = (unsigned char)(c->Nl >> 8);
+    p[sizeof(c->u) - 3] = (unsigned char)(c->Nl >> 16);
+    p[sizeof(c->u) - 4] = (unsigned char)(c->Nl >> 24);
+    p[sizeof(c->u) - 5] = (unsigned char)(c->Nl >> 32);
+    p[sizeof(c->u) - 6] = (unsigned char)(c->Nl >> 40);
+    p[sizeof(c->u) - 7] = (unsigned char)(c->Nl >> 48);
+    p[sizeof(c->u) - 8] = (unsigned char)(c->Nl >> 56);
+    p[sizeof(c->u) - 9] = (unsigned char)(c->Nh);
+    p[sizeof(c->u) - 10] = (unsigned char)(c->Nh >> 8);
+    p[sizeof(c->u) - 11] = (unsigned char)(c->Nh >> 16);
+    p[sizeof(c->u) - 12] = (unsigned char)(c->Nh >> 24);
+    p[sizeof(c->u) - 13] = (unsigned char)(c->Nh >> 32);
+    p[sizeof(c->u) - 14] = (unsigned char)(c->Nh >> 40);
+    p[sizeof(c->u) - 15] = (unsigned char)(c->Nh >> 48);
+    p[sizeof(c->u) - 16] = (unsigned char)(c->Nh >> 56);
+#endif
+
+    sha512_block_data_order(c, p, 1);
+
+    switch (c->md_len) {
+    /* Let compiler decide if it's appropriate to unroll... */
+    case SHA224_DIGEST_LENGTH:
+        for (n = 0; n < SHA224_DIGEST_LENGTH / 8; n++) {
+            SHA_LONG64 t = c->h[n];
+
+            *(md++) = (unsigned char)(t >> 56);
+            *(md++) = (unsigned char)(t >> 48);
+            *(md++) = (unsigned char)(t >> 40);
+            *(md++) = (unsigned char)(t >> 32);
+            *(md++) = (unsigned char)(t >> 24);
+            *(md++) = (unsigned char)(t >> 16);
+            *(md++) = (unsigned char)(t >> 8);
+            *(md++) = (unsigned char)(t);
+        }
+        /*
+         * For 224 bits, there are four bytes left over that have to be
+         * processed separately.
+         */
+        {
+            SHA_LONG64 t = c->h[SHA224_DIGEST_LENGTH / 8];
+
+            *(md++) = (unsigned char)(t >> 56);
+            *(md++) = (unsigned char)(t >> 48);
+            *(md++) = (unsigned char)(t >> 40);
+            *(md++) = (unsigned char)(t >> 32);
+        }
+        break;
+    case SHA256_DIGEST_LENGTH:
+        for (n = 0; n < SHA256_DIGEST_LENGTH / 8; n++) {
+            SHA_LONG64 t = c->h[n];
+
+            *(md++) = (unsigned char)(t >> 56);
+            *(md++) = (unsigned char)(t >> 48);
+            *(md++) = (unsigned char)(t >> 40);
+            *(md++) = (unsigned char)(t >> 32);
+            *(md++) = (unsigned char)(t >> 24);
+            *(md++) = (unsigned char)(t >> 16);
+            *(md++) = (unsigned char)(t >> 8);
+            *(md++) = (unsigned char)(t);
+        }
+        break;
+    case SHA384_DIGEST_LENGTH:
+        for (n = 0; n < SHA384_DIGEST_LENGTH / 8; n++) {
+            SHA_LONG64 t = c->h[n];
+
+            *(md++) = (unsigned char)(t >> 56);
+            *(md++) = (unsigned char)(t >> 48);
+            *(md++) = (unsigned char)(t >> 40);
+            *(md++) = (unsigned char)(t >> 32);
+            *(md++) = (unsigned char)(t >> 24);
+            *(md++) = (unsigned char)(t >> 16);
+            *(md++) = (unsigned char)(t >> 8);
+            *(md++) = (unsigned char)(t);
+        }
+        break;
+    case SHA512_DIGEST_LENGTH:
+        for (n = 0; n < SHA512_DIGEST_LENGTH / 8; n++) {
+            SHA_LONG64 t = c->h[n];
+
+            *(md++) = (unsigned char)(t >> 56);
+            *(md++) = (unsigned char)(t >> 48);
+            *(md++) = (unsigned char)(t >> 40);
+            *(md++) = (unsigned char)(t >> 32);
+            *(md++) = (unsigned char)(t >> 24);
+            *(md++) = (unsigned char)(t >> 16);
+            *(md++) = (unsigned char)(t >> 8);
+            *(md++) = (unsigned char)(t);
+        }
+        break;
+    /* ... as well as make sure md_len is not abused. */
+    default:
+        __assert_unreachable();
+    }
+}
+
+static int
+ossl_sha512_update(void *c_, const void *_data, unsigned int len)
+{
+    SHA512_CTX *c = c_;
+    SHA_LONG64 l;
+    unsigned char *p = c->u.p;
+    const unsigned char *data = (const unsigned char *)_data;
+
+    if (len == 0)
+        return 0;
+
+    l = (c->Nl + (((SHA_LONG64) len) << 3)) & U64(0xffffffffffffffff);
+    if (l < c->Nl)
+        c->Nh++;
+    if (sizeof(len) >= 8)
+        c->Nh += (((SHA_LONG64) len) >> 61);
+    c->Nl = l;
+
+    if (c->num != 0) {
+        size_t n = sizeof(c->u) - c->num;
+
+        if (len < n) {
+            memcpy(p + c->num, data, len), c->num += (unsigned int)len;
+            return 0;
+        } else {
+            memcpy(p + c->num, data, n), c->num = 0;
+            len -= n, data += n;
+            sha512_block_data_order(c, p, 1);
+        }
+    }
+
+    if (len >= sizeof(c->u)) {
+#ifndef SHA512_BLOCK_CAN_MANAGE_UNALIGNED_DATA
+        if ((size_t)data % sizeof(c->u.d[0]) != 0)
+            while (len >= sizeof(c->u))
+                memcpy(p, data, sizeof(c->u)),
+                sha512_block_data_order(c, p, 1),
+                len -= sizeof(c->u), data += sizeof(c->u);
+        else
+#endif
+            sha512_block_data_order(c, data, len / sizeof(c->u)),
+            data += len, len %= sizeof(c->u), data -= len;
+    }
+
+    if (len != 0)
+        memcpy(p, data, len), c->num = (int)len;
+
+    return 0;
+}
+
+struct auth_hash ossl_hash_sha384 = {
+	.type = CRYPTO_SHA2_384,
+	.name = "OpenSSL-SHA2-384",
+	.hashsize = SHA2_384_HASH_LEN,
+	.ctxsize = sizeof(SHA512_CTX),
+	.blocksize = SHA2_384_BLOCK_LEN,
+	.Init = ossl_sha384_init,
+	.Update = ossl_sha512_update,
+	.Final = ossl_sha512_final,
+};
+
+struct auth_hash ossl_hash_sha512 = {
+	.type = CRYPTO_SHA2_512,
+	.name = "OpenSSL-SHA2-512",
+	.hashsize = SHA2_512_HASH_LEN,
+	.ctxsize = sizeof(SHA512_CTX),
+	.blocksize = SHA2_512_BLOCK_LEN,
+	.Init = ossl_sha512_init,
+	.Update = ossl_sha512_update,
+	.Final = ossl_sha512_final,
+};
+
+_Static_assert(sizeof(SHA512_CTX) <= sizeof(struct ossl_hash_context),
+    "ossl_hash_context too small");