Add a crypto capability flag for accelerated software drivers.

Use this in GELI to print out a different message when accelerated
software such as AESNI is used vs plain software crypto.

While here, simplify the logic in GELI a bit for determing which type
of crypto driver was chosen the first time by examining the
capabilities of the matched driver after a single call to
crypto_newsession rather than making separate calls with different
flags.

Reviewed by:	delphij
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D25126

9 files changed, 30 insertions, 16 deletions

diff --git a/share/man/man9/crypto_driver.9 b/share/man/man9/crypto_driver.9
index 5a205ee4a094..930cc0d8bde4 100644
--- a/share/man/man9/crypto_driver.9
+++ b/share/man/man9/crypto_driver.9
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 25, 2020
+.Dd June 9, 2020
 .Dt CRYPTO_DRIVER 9
 .Os
 .Sh NAME
@@ -113,6 +113,8 @@ should be used for drivers which process requests on separate co-processors.
 .Dv CRYPTOCAP_F_SYNC
 should be set for drivers which process requests synchronously in
 .Fn CRYPTODEV_PROCESS .
+.Dv CRYPTOCAP_F_ACCEL_SOFTWARE
+should be set for software drivers which use accelerated CPU instructions.
 .Fn crypto_get_driverid
 returns an opaque driver id.
 .Pp
diff --git a/sys/crypto/aesni/aesni.c b/sys/crypto/aesni/aesni.c
index 2ef0f7f39de5..38be9b0acb68 100644
--- a/sys/crypto/aesni/aesni.c
+++ b/sys/crypto/aesni/aesni.c
@@ -167,7 +167,8 @@ aesni_attach(device_t dev)
 	sc = device_get_softc(dev);
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session),
-	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
+	    CRYPTOCAP_F_ACCEL_SOFTWARE);
 	if (sc->cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c
index 21b3fa2f71dd..bc3e70f935b5 100644
--- a/sys/crypto/armv8/armv8_crypto.c
+++ b/sys/crypto/armv8/armv8_crypto.c
@@ -131,7 +131,7 @@ armv8_crypto_attach(device_t dev)
 	sc->dieing = 0;
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
-	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
 	if (sc->cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
diff --git a/sys/crypto/blake2/blake2_cryptodev.c b/sys/crypto/blake2/blake2_cryptodev.c
index 262823b5a758..065f53734e54 100644
--- a/sys/crypto/blake2/blake2_cryptodev.c
+++ b/sys/crypto/blake2/blake2_cryptodev.c
@@ -129,7 +129,8 @@ blake2_attach(device_t dev)
 	sc->dying = false;
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
-	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
+	    CRYPTOCAP_F_ACCEL_SOFTWARE);
 	if (sc->cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
diff --git a/sys/crypto/via/padlock.c b/sys/crypto/via/padlock.c
index e0cd452bc431..2b26b14c6461 100644
--- a/sys/crypto/via/padlock.c
+++ b/sys/crypto/via/padlock.c
@@ -119,7 +119,8 @@ padlock_attach(device_t dev)
 	struct padlock_softc *sc = device_get_softc(dev);
 
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct padlock_session),
-	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
+	    CRYPTOCAP_F_ACCEL_SOFTWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index 2a7076c0fd28..8e7433f23594 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -489,7 +489,8 @@ g_eli_newsession(struct g_eli_worker *wr)
 {
 	struct g_eli_softc *sc;
 	struct crypto_session_params csp;
-	int error;
+	uint32_t caps;
+	int error, new_crypto;
 	void *key;
 
 	sc = wr->w_softc;
@@ -516,6 +517,7 @@ g_eli_newsession(struct g_eli_worker *wr)
 	}
 
 	switch (sc->sc_crypto) {
+	case G_ELI_CRYPTO_SW_ACCEL:
 	case G_ELI_CRYPTO_SW:
 		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_SOFTWARE);
@@ -526,18 +528,18 @@ g_eli_newsession(struct g_eli_worker *wr)
 		break;
 	case G_ELI_CRYPTO_UNKNOWN:
 		error = crypto_newsession(&wr->w_sid, &csp,
-		    CRYPTOCAP_F_HARDWARE);
+		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 		if (error == 0) {
+			caps = crypto_ses2caps(wr->w_sid);
+			if (caps & CRYPTOCAP_F_HARDWARE)
+				new_crypto = G_ELI_CRYPTO_HW;
+			else if (caps & CRYPTOCAP_F_ACCEL_SOFTWARE)
+				new_crypto = G_ELI_CRYPTO_SW_ACCEL;
+			else
+				new_crypto = G_ELI_CRYPTO_SW;
 			mtx_lock(&sc->sc_queue_mtx);
 			if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
-				sc->sc_crypto = G_ELI_CRYPTO_HW;
-			mtx_unlock(&sc->sc_queue_mtx);
-		} else {
-			error = crypto_newsession(&wr->w_sid, &csp,
-			    CRYPTOCAP_F_SOFTWARE);
-			mtx_lock(&sc->sc_queue_mtx);
-			if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
-				sc->sc_crypto = G_ELI_CRYPTO_SW;
+				sc->sc_crypto = new_crypto;
 			mtx_unlock(&sc->sc_queue_mtx);
 		}
 		break;
@@ -983,6 +985,7 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 	if (sc->sc_flags & G_ELI_FLAG_AUTH)
 		G_ELI_DEBUG(0, " Integrity: %s", g_eli_algo2str(sc->sc_aalgo));
 	G_ELI_DEBUG(0, "    Crypto: %s",
+	    sc->sc_crypto == G_ELI_CRYPTO_SW_ACCEL ? "accelerated software" :
 	    sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
 	return (gp);
 failed:
@@ -1381,6 +1384,9 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 	case G_ELI_CRYPTO_SW:
 		sbuf_cat(sb, "software");
 		break;
+	case G_ELI_CRYPTO_SW_ACCEL:
+		sbuf_cat(sb, "accelerated software");
+		break;
 	default:
 		sbuf_cat(sb, "UNKNOWN");
 		break;
diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h
index 1853aa17dbc1..e66bdaf409b6 100644
--- a/sys/geom/eli/g_eli.h
+++ b/sys/geom/eli/g_eli.h
@@ -145,6 +145,7 @@
 #define	G_ELI_CRYPTO_UNKNOWN	0
 #define	G_ELI_CRYPTO_HW		1
 #define	G_ELI_CRYPTO_SW		2
+#define	G_ELI_CRYPTO_SW_ACCEL	3
 
 #ifdef _KERNEL
 #if (MAX_KEY_BYTES < G_ELI_DATAIVKEYLEN)
diff --git a/sys/mips/cavium/cryptocteon/cryptocteon.c b/sys/mips/cavium/cryptocteon/cryptocteon.c
index 9e6bc0c358ea..d03b35db56b4 100644
--- a/sys/mips/cavium/cryptocteon/cryptocteon.c
+++ b/sys/mips/cavium/cryptocteon/cryptocteon.c
@@ -86,7 +86,8 @@ cryptocteon_attach(device_t dev)
 	sc = device_get_softc(dev);
 
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct octo_sess),
-	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
+	    CRYPTOCAP_F_ACCEL_SOFTWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "crypto_get_driverid ret %d\n", sc->sc_cid);
 		return (ENXIO);
diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h
index d51df5035731..836cb3b38acd 100644
--- a/sys/opencrypto/cryptodev.h
+++ b/sys/opencrypto/cryptodev.h
@@ -620,6 +620,7 @@ extern	void crypto_freesession(crypto_session_t cses);
 #define	CRYPTOCAP_F_HARDWARE	CRYPTO_FLAG_HARDWARE
 #define	CRYPTOCAP_F_SOFTWARE	CRYPTO_FLAG_SOFTWARE
 #define	CRYPTOCAP_F_SYNC	0x04000000	/* operates synchronously */
+#define	CRYPTOCAP_F_ACCEL_SOFTWARE 0x08000000
 extern	int32_t crypto_get_driverid(device_t dev, size_t session_size,
     int flags);
 extern	int crypto_find_driver(const char *);