diff options
Diffstat (limited to 'sys/opencrypto')
| -rw-r--r-- | sys/opencrypto/criov.c | 85 | ||||
| -rw-r--r-- | sys/opencrypto/crypto.c | 1309 | ||||
| -rw-r--r-- | sys/opencrypto/cryptodev.c | 555 | ||||
| -rw-r--r-- | sys/opencrypto/cryptodev.h | 144 | ||||
| -rw-r--r-- | sys/opencrypto/cryptodev_if.m | 118 | ||||
| -rw-r--r-- | sys/opencrypto/cryptosoft.c | 1687 | ||||
| -rw-r--r-- | sys/opencrypto/cryptosoft.h | 71 | ||||
| -rw-r--r-- | sys/opencrypto/ktls_ocf.c | 95 | ||||
| -rw-r--r-- | sys/opencrypto/xform_gmac.c | 6 |
9 files changed, 2382 insertions, 1688 deletions
diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c index d80894909130..e097a22713b3 100644 --- a/sys/opencrypto/criov.c +++ b/sys/opencrypto/criov.c @@ -157,41 +157,62 @@ cuio_apply(struct uio *uio, int off, int len, int (*f)(void *, void *, u_int), } void -crypto_copyback(int flags, caddr_t buf, int off, int size, c_caddr_t in) +crypto_copyback(struct cryptop *crp, int off, int size, const void *src) { - if ((flags & CRYPTO_F_IMBUF) != 0) - m_copyback((struct mbuf *)buf, off, size, in); - else if ((flags & CRYPTO_F_IOV) != 0) - cuio_copyback((struct uio *)buf, off, size, in); - else - bcopy(in, buf + off, size); + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + m_copyback(crp->crp_mbuf, off, size, src); + break; + case CRYPTO_BUF_UIO: + cuio_copyback(crp->crp_uio, off, size, src); + break; + case CRYPTO_BUF_CONTIG: + bcopy(src, crp->crp_buf + off, size); + break; + default: + panic("invalid crp buf type %d", crp->crp_buf_type); + } } void -crypto_copydata(int flags, caddr_t buf, int off, int size, caddr_t out) +crypto_copydata(struct cryptop *crp, int off, int size, void *dst) { - if ((flags & CRYPTO_F_IMBUF) != 0) - m_copydata((struct mbuf *)buf, off, size, out); - else if ((flags & CRYPTO_F_IOV) != 0) - cuio_copydata((struct uio *)buf, off, size, out); - else - bcopy(buf + off, out, size); + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + m_copydata(crp->crp_mbuf, off, size, dst); + break; + case CRYPTO_BUF_UIO: + cuio_copydata(crp->crp_uio, off, size, dst); + break; + case CRYPTO_BUF_CONTIG: + bcopy(crp->crp_buf + off, dst, size); + break; + default: + panic("invalid crp buf type %d", crp->crp_buf_type); + } } int -crypto_apply(int flags, caddr_t buf, int off, int len, +crypto_apply(struct cryptop *crp, int off, int len, int (*f)(void *, void *, u_int), void *arg) { int error; - if ((flags & CRYPTO_F_IMBUF) != 0) - error = m_apply((struct mbuf *)buf, off, len, f, arg); - else if ((flags & CRYPTO_F_IOV) != 0) - error = cuio_apply((struct uio *)buf, off, len, f, arg); - else - error = (*f)(arg, buf + off, len); + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + error = m_apply(crp->crp_mbuf, off, len, f, arg); + break; + case CRYPTO_BUF_UIO: + error = cuio_apply(crp->crp_uio, off, len, f, arg); + break; + case CRYPTO_BUF_CONTIG: + error = (*f)(arg, crp->crp_buf + off, len); + break; + default: + panic("invalid crp buf type %d", crp->crp_buf_type); + } return (error); } @@ -279,17 +300,17 @@ cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len) } void * -crypto_contiguous_subsegment(int crp_flags, void *crpbuf, - size_t skip, size_t len) +crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len) { - if ((crp_flags & CRYPTO_F_IMBUF) != 0) - return (m_contiguous_subsegment(crpbuf, skip, len)); - else if ((crp_flags & CRYPTO_F_IOV) != 0) - return (cuio_contiguous_segment(crpbuf, skip, len)); - else { - MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) != - (CRYPTO_F_IMBUF | CRYPTO_F_IOV)); - return ((char*)crpbuf + skip); + + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + return (m_contiguous_subsegment(crp->crp_mbuf, skip, len)); + case CRYPTO_BUF_UIO: + return (cuio_contiguous_segment(crp->crp_uio, skip, len)); + case CRYPTO_BUF_CONTIG: + return (crp->crp_buf + skip); + default: + panic("invalid crp buf type %d", crp->crp_buf_type); } } - diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index f3a3bba2e59b..4dde88fc6fea 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #define CRYPTO_TIMING /* enable timing support */ +#include "opt_compat.h" #include "opt_ddb.h" #include <sys/param.h> @@ -69,6 +70,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/malloc.h> #include <sys/proc.h> +#include <sys/refcount.h> #include <sys/sdt.h> #include <sys/smp.h> #include <sys/sysctl.h> @@ -79,7 +81,8 @@ __FBSDID("$FreeBSD$"); #include <vm/uma.h> #include <crypto/intake.h> #include <opencrypto/cryptodev.h> -#include <opencrypto/xform.h> /* XXX for M_XDATA */ +#include <opencrypto/xform_auth.h> +#include <opencrypto/xform_enc.h> #include <sys/kobj.h> #include <sys/bus.h> @@ -89,19 +92,12 @@ __FBSDID("$FreeBSD$"); #include <machine/pcb.h> #endif -struct crypto_session { - device_t parent; - void *softc; - uint32_t hid; - uint32_t capabilities; -}; - SDT_PROVIDER_DEFINE(opencrypto); /* * Crypto drivers register themselves by allocating a slot in the * crypto_drivers table with crypto_get_driverid() and then registering - * each algorithm they support with crypto_register() and crypto_kregister(). + * each asym algorithm they support with crypto_kregister(). */ static struct mtx crypto_drivers_mtx; /* lock on driver table */ #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) @@ -117,15 +113,10 @@ static struct mtx crypto_drivers_mtx; /* lock on driver table */ * Not tagged fields are read-only. */ struct cryptocap { - device_t cc_dev; /* (d) device/driver */ + device_t cc_dev; + uint32_t cc_hid; u_int32_t cc_sessions; /* (d) # of sessions */ u_int32_t cc_koperations; /* (d) # os asym operations */ - /* - * Largest possible operator length (in bits) for each type of - * encryption algorithm. XXX not used - */ - u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1]; - u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1]; u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; int cc_flags; /* (d) flags */ @@ -133,9 +124,17 @@ struct cryptocap { int cc_qblocked; /* (q) symmetric q blocked */ int cc_kqblocked; /* (q) asymmetric q blocked */ size_t cc_session_size; + volatile int cc_refs; +}; + +static struct cryptocap **crypto_drivers = NULL; +static int crypto_drivers_size = 0; + +struct crypto_session { + struct cryptocap *cap; + void *softc; + struct crypto_session_params csp; }; -static struct cryptocap *crypto_drivers = NULL; -static int crypto_drivers_num = 0; /* * There are two queues for crypto requests; one for symmetric (e.g. @@ -151,6 +150,9 @@ static struct mtx crypto_q_mtx; #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) +static SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0, + "In-kernel cryptography"); + /* * Taskqueue used to dispatch the crypto requests * that have the CRYPTO_F_ASYNC flag @@ -187,22 +189,37 @@ static struct crypto_ret_worker *crypto_ret_workers = NULL; (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q)) static int crypto_workers_num = 0; +SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN, + &crypto_workers_num, 0, + "Number of crypto workers used to dispatch crypto jobs"); +#ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN, &crypto_workers_num, 0, "Number of crypto workers used to dispatch crypto jobs"); +#endif static uma_zone_t cryptop_zone; -static uma_zone_t cryptodesc_zone; static uma_zone_t cryptoses_zone; -int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ +int crypto_userasymcrypto = 1; +SYSCTL_INT(_kern_crypto, OID_AUTO, asym_enable, CTLFLAG_RW, + &crypto_userasymcrypto, 0, + "Enable user-mode access to asymmetric crypto support"); +#ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, &crypto_userasymcrypto, 0, "Enable/disable user-mode access to asymmetric crypto support"); -int crypto_devallowsoft = 0; /* only use hardware crypto */ +#endif + +int crypto_devallowsoft = 0; +SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RW, + &crypto_devallowsoft, 0, + "Enable use of software crypto by /dev/crypto"); +#ifdef COMPAT_FREEBSD12 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, &crypto_devallowsoft, 0, "Enable/disable use of software crypto by /dev/crypto"); +#endif MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); @@ -211,13 +228,12 @@ static struct proc *cryptoproc; static void crypto_ret_proc(struct crypto_ret_worker *ret_worker); static void crypto_destroy(void); static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); -static int crypto_kinvoke(struct cryptkop *krp, int flags); -static void crypto_remove(struct cryptocap *cap); +static int crypto_kinvoke(struct cryptkop *krp); static void crypto_task_invoke(void *ctx, int pending); static void crypto_batch_enqueue(struct cryptop *crp); static struct cryptostats cryptostats; -SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, +SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats, cryptostats, "Crypto system statistics"); #ifdef CRYPTO_TIMING @@ -257,6 +273,29 @@ struct keybuf * get_keybuf(void) { return (keybuf); } +static struct cryptocap * +cap_ref(struct cryptocap *cap) +{ + + refcount_acquire(&cap->cc_refs); + return (cap); +} + +static void +cap_rele(struct cryptocap *cap) +{ + + if (refcount_release(&cap->cc_refs) == 0) + return; + + KASSERT(cap->cc_sessions == 0, + ("freeing crypto driver with active sessions")); + KASSERT(cap->cc_koperations == 0, + ("freeing crypto driver with active key operations")); + + free(cap, M_CRYPTO_DATA); +} + static int crypto_init(void) { @@ -273,22 +312,18 @@ crypto_init(void) cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 0, 0, 0, 0, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); - cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), - 0, 0, 0, 0, - UMA_ALIGN_PTR, UMA_ZONE_ZINIT); cryptoses_zone = uma_zcreate("crypto_session", sizeof(struct crypto_session), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_ZINIT); - if (cryptodesc_zone == NULL || cryptop_zone == NULL || - cryptoses_zone == NULL) { + if (cryptop_zone == NULL || cryptoses_zone == NULL) { printf("crypto_init: cannot setup crypto zones\n"); error = ENOMEM; goto bad; } - crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; - crypto_drivers = malloc(crypto_drivers_num * + crypto_drivers_size = CRYPTO_DRIVERS_INITIAL; + crypto_drivers = malloc(crypto_drivers_size * sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); if (crypto_drivers == NULL) { printf("crypto_init: cannot setup crypto drivers\n"); @@ -380,9 +415,56 @@ crypto_terminate(struct proc **pp, void *q) } static void +hmac_init_pad(struct auth_hash *axf, const char *key, int klen, void *auth_ctx, + uint8_t padval) +{ + uint8_t hmac_key[HMAC_MAX_BLOCK_LEN]; + u_int i; + + KASSERT(axf->blocksize <= sizeof(hmac_key), + ("Invalid HMAC block size %d", axf->blocksize)); + + /* + * If the key is larger than the block size, use the digest of + * the key as the key instead. + */ + memset(hmac_key, 0, sizeof(hmac_key)); + if (klen > axf->blocksize) { + axf->Init(auth_ctx); + axf->Update(auth_ctx, key, klen); + axf->Final(hmac_key, auth_ctx); + klen = axf->hashsize; + } else + memcpy(hmac_key, key, klen); + + for (i = 0; i < axf->blocksize; i++) + hmac_key[i] ^= padval; + + axf->Init(auth_ctx); + axf->Update(auth_ctx, hmac_key, axf->blocksize); +} + +void +hmac_init_ipad(struct auth_hash *axf, const char *key, int klen, + void *auth_ctx) +{ + + hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL); +} + +void +hmac_init_opad(struct auth_hash *axf, const char *key, int klen, + void *auth_ctx) +{ + + hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL); +} + +static void crypto_destroy(void) { struct crypto_ret_worker *ret_worker; + int i; /* * Terminate any crypto threads. @@ -400,13 +482,14 @@ crypto_destroy(void) /* * Reclaim dynamically allocated resources. */ - if (crypto_drivers != NULL) - free(crypto_drivers, M_CRYPTO_DATA); + for (i = 0; i < crypto_drivers_size; i++) { + if (crypto_drivers[i] != NULL) + cap_rele(crypto_drivers[i]); + } + free(crypto_drivers, M_CRYPTO_DATA); if (cryptoses_zone != NULL) uma_zdestroy(cryptoses_zone); - if (cryptodesc_zone != NULL) - uma_zdestroy(cryptodesc_zone); if (cryptop_zone != NULL) uma_zdestroy(cryptop_zone); mtx_destroy(&crypto_q_mtx); @@ -421,13 +504,13 @@ crypto_destroy(void) uint32_t crypto_ses2hid(crypto_session_t crypto_session) { - return (crypto_session->hid); + return (crypto_session->cap->cc_hid); } uint32_t crypto_ses2caps(crypto_session_t crypto_session) { - return (crypto_session->capabilities); + return (crypto_session->cap->cc_flags & 0xff000000); } void * @@ -436,86 +519,411 @@ crypto_get_driver_session(crypto_session_t crypto_session) return (crypto_session->softc); } -static struct cryptocap * -crypto_checkdriver(u_int32_t hid) +const struct crypto_session_params * +crypto_get_params(crypto_session_t crypto_session) { - if (crypto_drivers == NULL) - return NULL; - return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); + return (&crypto_session->csp); } -/* - * Compare a driver's list of supported algorithms against another - * list; return non-zero if all algorithms are supported. - */ -static int -driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri) +struct auth_hash * +crypto_auth_hash(const struct crypto_session_params *csp) +{ + + switch (csp->csp_auth_alg) { + case CRYPTO_MD5_HMAC: + return (&auth_hash_hmac_md5); + case CRYPTO_SHA1_HMAC: + return (&auth_hash_hmac_sha1); + case CRYPTO_SHA2_224_HMAC: + return (&auth_hash_hmac_sha2_224); + case CRYPTO_SHA2_256_HMAC: + return (&auth_hash_hmac_sha2_256); + case CRYPTO_SHA2_384_HMAC: + return (&auth_hash_hmac_sha2_384); + case CRYPTO_SHA2_512_HMAC: + return (&auth_hash_hmac_sha2_512); + case CRYPTO_NULL_HMAC: + return (&auth_hash_null); + case CRYPTO_RIPEMD160_HMAC: + return (&auth_hash_hmac_ripemd_160); + case CRYPTO_MD5_KPDK: + return (&auth_hash_key_md5); + case CRYPTO_SHA1_KPDK: + return (&auth_hash_key_sha1); +#ifdef notyet + case CRYPTO_MD5: + return (&auth_hash_md5); +#endif + case CRYPTO_SHA1: + return (&auth_hash_sha1); + case CRYPTO_SHA2_224: + return (&auth_hash_sha2_224); + case CRYPTO_SHA2_256: + return (&auth_hash_sha2_256); + case CRYPTO_SHA2_384: + return (&auth_hash_sha2_384); + case CRYPTO_SHA2_512: + return (&auth_hash_sha2_512); + case CRYPTO_AES_NIST_GMAC: + switch (csp->csp_auth_klen) { + case 128 / 8: + return (&auth_hash_nist_gmac_aes_128); + case 192 / 8: + return (&auth_hash_nist_gmac_aes_192); + case 256 / 8: + return (&auth_hash_nist_gmac_aes_256); + default: + return (NULL); + } + case CRYPTO_BLAKE2B: + return (&auth_hash_blake2b); + case CRYPTO_BLAKE2S: + return (&auth_hash_blake2s); + case CRYPTO_POLY1305: + return (&auth_hash_poly1305); + case CRYPTO_AES_CCM_CBC_MAC: + switch (csp->csp_auth_klen) { + case 128 / 8: + return (&auth_hash_ccm_cbc_mac_128); + case 192 / 8: + return (&auth_hash_ccm_cbc_mac_192); + case 256 / 8: + return (&auth_hash_ccm_cbc_mac_256); + default: + return (NULL); + } + default: + return (NULL); + } +} + +struct enc_xform * +crypto_cipher(const struct crypto_session_params *csp) { - const struct cryptoini *cr; - /* See if all the algorithms are supported. */ - for (cr = cri; cr; cr = cr->cri_next) - if (cap->cc_alg[cr->cri_alg] == 0) - return 0; - return 1; + switch (csp->csp_cipher_alg) { + case CRYPTO_DES_CBC: + return (&enc_xform_des); + case CRYPTO_3DES_CBC: + return (&enc_xform_3des); + case CRYPTO_BLF_CBC: + return (&enc_xform_blf); + case CRYPTO_CAST_CBC: + return (&enc_xform_cast5); + case CRYPTO_SKIPJACK_CBC: + return (&enc_xform_skipjack); + case CRYPTO_RIJNDAEL128_CBC: + return (&enc_xform_rijndael128); + case CRYPTO_AES_XTS: + return (&enc_xform_aes_xts); + case CRYPTO_AES_ICM: + return (&enc_xform_aes_icm); + case CRYPTO_AES_NIST_GCM_16: + return (&enc_xform_aes_nist_gcm); + case CRYPTO_CAMELLIA_CBC: + return (&enc_xform_camellia); + case CRYPTO_NULL_CBC: + return (&enc_xform_null); + case CRYPTO_CHACHA20: + return (&enc_xform_chacha20); + case CRYPTO_AES_CCM_16: + return (&enc_xform_ccm); + default: + return (NULL); + } +} + +static struct cryptocap * +crypto_checkdriver(u_int32_t hid) +{ + + return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]); } /* * Select a driver for a new session that supports the specified * algorithms and, optionally, is constrained according to the flags. - * The algorithm we use here is pretty stupid; just use the - * first driver that supports all the algorithms we need. If there - * are multiple drivers we choose the driver with the fewest active - * sessions. We prefer hardware-backed drivers to software ones. - * - * XXX We need more smarts here (in real life too, but that's - * XXX another story altogether). */ static struct cryptocap * -crypto_select_driver(const struct cryptoini *cri, int flags) +crypto_select_driver(const struct crypto_session_params *csp, int flags) { struct cryptocap *cap, *best; - int match, hid; + int best_match, error, hid; CRYPTO_DRIVER_ASSERT(); - /* - * Look first for hardware crypto devices if permitted. - */ - if (flags & CRYPTOCAP_F_HARDWARE) - match = CRYPTOCAP_F_HARDWARE; - else - match = CRYPTOCAP_F_SOFTWARE; best = NULL; -again: - for (hid = 0; hid < crypto_drivers_num; hid++) { - cap = &crypto_drivers[hid]; + for (hid = 0; hid < crypto_drivers_size; hid++) { /* - * If it's not initialized, is in the process of - * going away, or is not appropriate (hardware - * or software based on match), then skip. + * If there is no driver for this slot, or the driver + * is not appropriate (hardware or software based on + * match), then skip. */ - if (cap->cc_dev == NULL || - (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || - (cap->cc_flags & match) == 0) + cap = crypto_drivers[hid]; + if (cap == NULL || + (cap->cc_flags & flags) == 0) continue; - /* verify all the algorithms are supported. */ - if (driver_suitable(cap, cri)) { - if (best == NULL || - cap->cc_sessions < best->cc_sessions) - best = cap; + error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp); + if (error >= 0) + continue; + + /* + * Use the driver with the highest probe value. + * Hardware drivers use a higher probe value than + * software. In case of a tie, prefer the driver with + * the fewest active sessions. + */ + if (best == NULL || error > best_match || + (error == best_match && + cap->cc_sessions < best->cc_sessions)) { + best = cap; + best_match = error; } } - if (best == NULL && match == CRYPTOCAP_F_HARDWARE && - (flags & CRYPTOCAP_F_SOFTWARE)) { - /* sort of an Algol 68-style for loop */ - match = CRYPTOCAP_F_SOFTWARE; - goto again; - } return best; } +static bool +alg_is_compression(int alg) +{ + + if (alg == CRYPTO_DEFLATE_COMP) + return (true); + return (false); +} + +static bool +alg_is_cipher(int alg) +{ + + if (alg >= CRYPTO_DES_CBC && alg <= CRYPTO_SKIPJACK_CBC) + return (true); + if (alg >= CRYPTO_AES_CBC && alg <= CRYPTO_ARC4) + return (true); + if (alg == CRYPTO_NULL_CBC) + return (true); + if (alg >= CRYPTO_CAMELLIA_CBC && alg <= CRYPTO_AES_ICM) + return (true); + if (alg == CRYPTO_CHACHA20) + return (true); + return (false); +} + +static bool +alg_is_digest(int alg) +{ + + if (alg >= CRYPTO_MD5_HMAC && alg <= CRYPTO_SHA1_KPDK) + return (true); + if (alg >= CRYPTO_MD5 && alg <= CRYPTO_SHA1) + return (true); + if (alg == CRYPTO_NULL_HMAC) + return (true); + if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC) + return (true); + if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC) + return (true); + if (alg == CRYPTO_AES_NIST_GMAC) + return (true); + if (alg >= CRYPTO_BLAKE2B && alg <= CRYPTO_BLAKE2S) + return (true); + if (alg >= CRYPTO_SHA2_224_HMAC && alg <= CRYPTO_POLY1305) + return (true); + if (alg == CRYPTO_AES_CCM_CBC_MAC) + return (true); + return (false); +} + +static bool +alg_is_keyed_digest(int alg) +{ + + if (alg >= CRYPTO_MD5_HMAC && alg <= CRYPTO_SHA1_KPDK) + return (true); + if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC) + return (true); + if (alg == CRYPTO_AES_NIST_GMAC) + return (true); + if (alg >= CRYPTO_BLAKE2B && alg <= CRYPTO_BLAKE2S) + return (true); + if (alg == CRYPTO_SHA2_224_HMAC) + return (true); + if (alg == CRYPTO_POLY1305) + return (true); + if (alg == CRYPTO_AES_CCM_CBC_MAC) + return (true); + return (false); +} + +static bool +alg_is_aead(int alg) +{ + + if (alg == CRYPTO_AES_NIST_GCM_16) + return (true); + if (alg == CRYPTO_AES_CCM_16) + return (true); + return (false); +} + +/* Various sanity checks on crypto session parameters. */ +static bool +check_csp(const struct crypto_session_params *csp) +{ + struct auth_hash *axf; + + /* Mode-independent checks. */ + if (csp->csp_flags != 0) + return (false); + if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 || + csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0) + return (false); + if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0) + return (false); + if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0) + return (false); + + switch (csp->csp_mode) { + case CSP_MODE_COMPRESS: + if (!alg_is_compression(csp->csp_cipher_alg)) + return (false); + if (csp->csp_flags != 0) + return (false); + if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 || + csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 || + csp->csp_auth_mlen != 0) + return (false); + break; + case CSP_MODE_CIPHER: + if (!alg_is_cipher(csp->csp_cipher_alg)) + return (false); + if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) { + if (csp->csp_cipher_klen == 0) + return (false); + if (csp->csp_cipher_alg != CRYPTO_ARC4) { + if (csp->csp_ivlen == 0) + return (false); + } + } + if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) + return (false); + if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 || + csp->csp_auth_mlen != 0) + return (false); + break; + case CSP_MODE_DIGEST: + if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0) + return (false); + + /* IV is optional for digests (e.g. GMAC). */ + if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) + return (false); + if (!alg_is_digest(csp->csp_auth_alg)) + return (false); + + /* Key is optional for BLAKE2 digests. */ + if (csp->csp_auth_alg == CRYPTO_BLAKE2B || + csp->csp_auth_alg == CRYPTO_BLAKE2S) + ; + else if (alg_is_keyed_digest(csp->csp_auth_alg)) { + if (csp->csp_auth_klen == 0) + return (false); + } else { + if (csp->csp_auth_klen != 0) + return (false); + } + if (csp->csp_auth_mlen != 0) { + axf = crypto_auth_hash(csp); + if (axf == NULL || csp->csp_auth_mlen > axf->hashsize) + return (false); + } + break; + case CSP_MODE_AEAD: + if (!alg_is_aead(csp->csp_cipher_alg)) + return (false); + if (csp->csp_cipher_klen == 0) + return (false); + if (csp->csp_ivlen == 0 || + csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) + return (false); + if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0) + return (false); + + /* + * XXX: Would be nice to have a better way to get this + * value. + */ + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: + if (csp->csp_auth_mlen > 16) + return (false); + break; + } + break; + case CSP_MODE_ETA: + if (!alg_is_cipher(csp->csp_cipher_alg)) + return (false); + if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) { + if (csp->csp_cipher_klen == 0) + return (false); + if (csp->csp_cipher_alg != CRYPTO_ARC4) { + if (csp->csp_ivlen == 0) + return (false); + } + } + if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN) + return (false); + if (!alg_is_digest(csp->csp_auth_alg)) + return (false); + + /* Key is optional for BLAKE2 digests. */ + if (csp->csp_auth_alg == CRYPTO_BLAKE2B || + csp->csp_auth_alg == CRYPTO_BLAKE2S) + ; + else if (alg_is_keyed_digest(csp->csp_auth_alg)) { + if (csp->csp_auth_klen == 0) + return (false); + } else { + if (csp->csp_auth_klen != 0) + return (false); + } + if (csp->csp_auth_mlen != 0) { + axf = crypto_auth_hash(csp); + if (axf == NULL || csp->csp_auth_mlen > axf->hashsize) + return (false); + } + break; + default: + return (false); + } + + return (true); +} + +/* + * Delete a session after it has been detached from its driver. + */ +static void +crypto_deletesession(crypto_session_t cses) +{ + struct cryptocap *cap; + + cap = cses->cap; + + explicit_bzero(cses->softc, cap->cc_session_size); + free(cses->softc, M_CRYPTO_DATA); + uma_zfree(cryptoses_zone, cses); + + CRYPTO_DRIVER_LOCK(); + cap->cc_sessions--; + if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP) + wakeup(cap); + CRYPTO_DRIVER_UNLOCK(); + cap_rele(cap); +} + /* * Create a new session. The crid argument specifies a crypto * driver to use or constraints on a driver to select (hardware @@ -523,18 +931,17 @@ again: * must be capable of the requested crypto algorithms. */ int -crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int crid) +crypto_newsession(crypto_session_t *cses, + const struct crypto_session_params *csp, int crid) { crypto_session_t res; - void *softc_mem; struct cryptocap *cap; - u_int32_t hid; - size_t softc_size; int err; -restart: + if (!check_csp(csp)) + return (EINVAL); + res = NULL; - softc_mem = NULL; CRYPTO_DRIVER_LOCK(); if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { @@ -542,76 +949,39 @@ restart: * Use specified driver; verify it is capable. */ cap = crypto_checkdriver(crid); - if (cap != NULL && !driver_suitable(cap, cri)) + if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0) cap = NULL; } else { /* * No requested driver; select based on crid flags. */ - cap = crypto_select_driver(cri, crid); - /* - * if NULL then can't do everything in one session. - * XXX Fix this. We need to inject a "virtual" session - * XXX layer right about here. - */ + cap = crypto_select_driver(csp, crid); } if (cap == NULL) { + CRYPTO_DRIVER_UNLOCK(); CRYPTDEB("no driver"); - err = EOPNOTSUPP; - goto out; + return (EOPNOTSUPP); } + cap_ref(cap); cap->cc_sessions++; - softc_size = cap->cc_session_size; - hid = cap - crypto_drivers; - cap = NULL; CRYPTO_DRIVER_UNLOCK(); - softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO); res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO); - res->softc = softc_mem; - - CRYPTO_DRIVER_LOCK(); - cap = crypto_checkdriver(hid); - if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0) { - cap->cc_sessions--; - crypto_remove(cap); - cap = NULL; - } - if (cap == NULL) { - free(softc_mem, M_CRYPTO_DATA); - uma_zfree(cryptoses_zone, res); - CRYPTO_DRIVER_UNLOCK(); - goto restart; - } + res->cap = cap; + res->softc = malloc(cap->cc_session_size, M_CRYPTO_DATA, M_WAITOK | + M_ZERO); + res->csp = *csp; /* Call the driver initialization routine. */ - err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, cri); + err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp); if (err != 0) { CRYPTDEB("dev newsession failed: %d", err); - goto out; + crypto_deletesession(res); + return (err); } - res->capabilities = cap->cc_flags & 0xff000000; - res->hid = hid; *cses = res; - -out: - CRYPTO_DRIVER_UNLOCK(); - if (err != 0) { - free(softc_mem, M_CRYPTO_DATA); - if (res != NULL) - uma_zfree(cryptoses_zone, res); - } - return err; -} - -static void -crypto_remove(struct cryptocap *cap) -{ - - mtx_assert(&crypto_drivers_mtx, MA_OWNED); - if (cap->cc_sessions == 0 && cap->cc_koperations == 0) - bzero(cap, sizeof(*cap)); + return (0); } /* @@ -622,101 +992,81 @@ void crypto_freesession(crypto_session_t cses) { struct cryptocap *cap; - void *ses; - size_t ses_size; - u_int32_t hid; if (cses == NULL) return; - CRYPTO_DRIVER_LOCK(); - - hid = crypto_ses2hid(cses); - KASSERT(hid < crypto_drivers_num, - ("bogus crypto_session %p hid %u", cses, hid)); - cap = &crypto_drivers[hid]; - - ses = cses->softc; - ses_size = cap->cc_session_size; - - if (cap->cc_sessions) - cap->cc_sessions--; + cap = cses->cap; /* Call the driver cleanup routine, if available. */ CRYPTODEV_FREESESSION(cap->cc_dev, cses); - explicit_bzero(ses, ses_size); - free(ses, M_CRYPTO_DATA); - uma_zfree(cryptoses_zone, cses); - - if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) - crypto_remove(cap); - - CRYPTO_DRIVER_UNLOCK(); + crypto_deletesession(cses); } /* - * Return an unused driver id. Used by drivers prior to registering - * support for the algorithms they handle. + * Return a new driver id. Registers a driver with the system so that + * it can be probed by subsequent sessions. */ int32_t crypto_get_driverid(device_t dev, size_t sessionsize, int flags) { - struct cryptocap *newdrv; + struct cryptocap *cap, **newdrv; int i; if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { - printf("%s: no flags specified when registering driver\n", - device_get_nameunit(dev)); + device_printf(dev, + "no flags specified when registering driver\n"); return -1; } + cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO); + cap->cc_dev = dev; + cap->cc_session_size = sessionsize; + cap->cc_flags = flags; + refcount_init(&cap->cc_refs, 1); + CRYPTO_DRIVER_LOCK(); + for (;;) { + for (i = 0; i < crypto_drivers_size; i++) { + if (crypto_drivers[i] == NULL) + break; + } - for (i = 0; i < crypto_drivers_num; i++) { - if (crypto_drivers[i].cc_dev == NULL && - (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { + if (i < crypto_drivers_size) break; - } - } - /* Out of entries, allocate some more. */ - if (i == crypto_drivers_num) { - /* Be careful about wrap-around. */ - if (2 * crypto_drivers_num <= crypto_drivers_num) { + /* Out of entries, allocate some more. */ + + if (2 * crypto_drivers_size <= crypto_drivers_size) { CRYPTO_DRIVER_UNLOCK(); printf("crypto: driver count wraparound!\n"); - return -1; + cap_rele(cap); + return (-1); } + CRYPTO_DRIVER_UNLOCK(); - newdrv = malloc(2 * crypto_drivers_num * - sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); - if (newdrv == NULL) { - CRYPTO_DRIVER_UNLOCK(); - printf("crypto: no space to expand driver table!\n"); - return -1; - } + newdrv = malloc(2 * crypto_drivers_size * + sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO); - bcopy(crypto_drivers, newdrv, - crypto_drivers_num * sizeof(struct cryptocap)); + CRYPTO_DRIVER_LOCK(); + memcpy(newdrv, crypto_drivers, + crypto_drivers_size * sizeof(*crypto_drivers)); - crypto_drivers_num *= 2; + crypto_drivers_size *= 2; free(crypto_drivers, M_CRYPTO_DATA); crypto_drivers = newdrv; } - /* NB: state is zero'd on free */ - crypto_drivers[i].cc_sessions = 1; /* Mark */ - crypto_drivers[i].cc_dev = dev; - crypto_drivers[i].cc_flags = flags; - crypto_drivers[i].cc_session_size = sessionsize; + cap->cc_hid = i; + crypto_drivers[i] = cap; + CRYPTO_DRIVER_UNLOCK(); + if (bootverbose) printf("crypto: assign %s driver id %u, flags 0x%x\n", device_get_nameunit(dev), i, flags); - CRYPTO_DRIVER_UNLOCK(); - return i; } @@ -729,20 +1079,22 @@ crypto_get_driverid(device_t dev, size_t sessionsize, int flags) int crypto_find_driver(const char *match) { + struct cryptocap *cap; int i, len = strlen(match); CRYPTO_DRIVER_LOCK(); - for (i = 0; i < crypto_drivers_num; i++) { - device_t dev = crypto_drivers[i].cc_dev; - if (dev == NULL || - (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP)) + for (i = 0; i < crypto_drivers_size; i++) { + if (crypto_drivers[i] == NULL) continue; - if (strncmp(match, device_get_nameunit(dev), len) == 0 || - strncmp(match, device_get_name(dev), len) == 0) - break; + cap = crypto_drivers[i]; + if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 || + strncmp(match, device_get_name(cap->cc_dev), len) == 0) { + CRYPTO_DRIVER_UNLOCK(); + return (i); + } } CRYPTO_DRIVER_UNLOCK(); - return i < crypto_drivers_num ? i : -1; + return (-1); } /* @@ -752,8 +1104,16 @@ crypto_find_driver(const char *match) device_t crypto_find_device_byhid(int hid) { - struct cryptocap *cap = crypto_checkdriver(hid); - return cap != NULL ? cap->cc_dev : NULL; + struct cryptocap *cap; + device_t dev; + + dev = NULL; + CRYPTO_DRIVER_LOCK(); + cap = crypto_checkdriver(hid); + if (cap != NULL) + dev = cap->cc_dev; + CRYPTO_DRIVER_UNLOCK(); + return (dev); } /* @@ -762,8 +1122,16 @@ crypto_find_device_byhid(int hid) int crypto_getcaps(int hid) { - struct cryptocap *cap = crypto_checkdriver(hid); - return cap != NULL ? cap->cc_flags : 0; + struct cryptocap *cap; + int flags; + + flags = 0; + CRYPTO_DRIVER_LOCK(); + cap = crypto_checkdriver(hid); + if (cap != NULL) + flags = cap->cc_flags; + CRYPTO_DRIVER_UNLOCK(); + return (flags); } /* @@ -803,103 +1171,6 @@ crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags) } /* - * Register support for a non-key-related algorithm. This routine - * is called once for each such algorithm supported by a driver. - */ -int -crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, - u_int32_t flags) -{ - struct cryptocap *cap; - int err; - - CRYPTO_DRIVER_LOCK(); - - cap = crypto_checkdriver(driverid); - /* NB: algorithms are in the range [1..max] */ - if (cap != NULL && - (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { - /* - * XXX Do some performance testing to determine placing. - * XXX We probably need an auxiliary data structure that - * XXX describes relative performances. - */ - - cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; - cap->cc_max_op_len[alg] = maxoplen; - if (bootverbose) - printf("crypto: %s registers alg %u flags %u maxoplen %u\n" - , device_get_nameunit(cap->cc_dev) - , alg - , flags - , maxoplen - ); - cap->cc_sessions = 0; /* Unmark */ - err = 0; - } else - err = EINVAL; - - CRYPTO_DRIVER_UNLOCK(); - return err; -} - -static void -driver_finis(struct cryptocap *cap) -{ - u_int32_t ses, kops; - - CRYPTO_DRIVER_ASSERT(); - - ses = cap->cc_sessions; - kops = cap->cc_koperations; - bzero(cap, sizeof(*cap)); - if (ses != 0 || kops != 0) { - /* - * If there are pending sessions, - * just mark as invalid. - */ - cap->cc_flags |= CRYPTOCAP_F_CLEANUP; - cap->cc_sessions = ses; - cap->cc_koperations = kops; - } -} - -/* - * Unregister a crypto driver. If there are pending sessions using it, - * leave enough information around so that subsequent calls using those - * sessions will correctly detect the driver has been unregistered and - * reroute requests. - */ -int -crypto_unregister(u_int32_t driverid, int alg) -{ - struct cryptocap *cap; - int i, err; - - CRYPTO_DRIVER_LOCK(); - cap = crypto_checkdriver(driverid); - if (cap != NULL && - (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && - cap->cc_alg[alg] != 0) { - cap->cc_alg[alg] = 0; - cap->cc_max_op_len[alg] = 0; - - /* Was this the last algorithm ? */ - for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) - if (cap->cc_alg[i] != 0) - break; - - if (i == CRYPTO_ALGORITHM_MAX + 1) - driver_finis(cap); - err = 0; - } else - err = EINVAL; - CRYPTO_DRIVER_UNLOCK(); - - return err; -} - -/* * Unregister all algorithms associated with a crypto driver. * If there are pending sessions using it, leave enough information * around so that subsequent calls using those sessions will @@ -910,18 +1181,27 @@ int crypto_unregister_all(u_int32_t driverid) { struct cryptocap *cap; - int err; CRYPTO_DRIVER_LOCK(); cap = crypto_checkdriver(driverid); - if (cap != NULL) { - driver_finis(cap); - err = 0; - } else - err = EINVAL; + if (cap == NULL) { + CRYPTO_DRIVER_UNLOCK(); + return (EINVAL); + } + + cap->cc_flags |= CRYPTOCAP_F_CLEANUP; + crypto_drivers[driverid] = NULL; + + /* + * XXX: This doesn't do anything to kick sessions that + * have no pending operations. + */ + while (cap->cc_sessions != 0 || cap->cc_koperations != 0) + mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0); CRYPTO_DRIVER_UNLOCK(); + cap_rele(cap); - return err; + return (0); } /* @@ -951,6 +1231,125 @@ crypto_unblock(u_int32_t driverid, int what) return err; } +#ifdef INVARIANTS +/* Various sanity checks on crypto requests. */ +static void +crp_sanity(struct cryptop *crp) +{ + struct crypto_session_params *csp; + + KASSERT(crp->crp_session != NULL, ("incoming crp without a session")); + KASSERT(crp->crp_ilen >= 0, ("incoming crp with -ve input length")); + KASSERT(crp->crp_etype == 0, ("incoming crp with error")); + KASSERT(!(crp->crp_flags & CRYPTO_F_DONE), + ("incoming crp already done")); + + csp = &crp->crp_session->csp; + switch (csp->csp_mode) { + case CSP_MODE_COMPRESS: + KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS || + crp->crp_op == CRYPTO_OP_DECOMPRESS, + ("invalid compression op %x", crp->crp_op)); + break; + case CSP_MODE_CIPHER: + KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT || + crp->crp_op == CRYPTO_OP_DECRYPT, + ("invalid cipher op %x", crp->crp_op)); + break; + case CSP_MODE_DIGEST: + KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST || + crp->crp_op == CRYPTO_OP_VERIFY_DIGEST, + ("invalid digest op %x", crp->crp_op)); + break; + case CSP_MODE_AEAD: + KASSERT(crp->crp_op == + (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) || + crp->crp_op == + (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST), + ("invalid AEAD op %x", crp->crp_op)); + if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) + KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE, + ("GCM without a separate IV")); + if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) + KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE, + ("CCM without a separate IV")); + break; + case CSP_MODE_ETA: + KASSERT(crp->crp_op == + (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) || + crp->crp_op == + (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST), + ("invalid ETA op %x", crp->crp_op)); + break; + } + KASSERT((crp->crp_flags & CRYPTO_F_IV_GENERATE) == 0 || + crp->crp_op == CRYPTO_OP_ENCRYPT || + crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST), + ("IV_GENERATE set for non-encryption operation %x", crp->crp_op)); + KASSERT((crp->crp_flags & + (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE)) != + (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE), + ("crp with both IV_SEPARATE and IV_GENERATE set")); + KASSERT(crp->crp_buf_type >= CRYPTO_BUF_CONTIG && + crp->crp_buf_type <= CRYPTO_BUF_MBUF, + ("invalid crp buffer type %d", crp->crp_buf_type)); + if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) { + KASSERT(crp->crp_aad_start == 0 || + crp->crp_aad_start < crp->crp_ilen, + ("invalid AAD start")); + KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0, + ("AAD with zero length and non-zero start")); + KASSERT(crp->crp_aad_length == 0 || + crp->crp_aad_start + crp->crp_aad_length <= crp->crp_ilen, + ("AAD outside input length")); + } else { + KASSERT(crp->crp_aad_start == 0 && crp->crp_aad_length == 0, + ("AAD region in request not supporting AAD")); + } + if (csp->csp_ivlen == 0) { + KASSERT((crp->crp_flags & + (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE)) == 0, + ("IV_GENERATE or IV_SEPARATE set when IV isn't used")); + KASSERT(crp->crp_iv_start == 0, + ("crp_iv_start set when IV isn't used")); + } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) { + KASSERT(crp->crp_iv_start == 0, + ("IV_SEPARATE used with non-zero IV start")); + } else { + KASSERT(crp->crp_iv_start < crp->crp_ilen, + ("invalid IV start")); + KASSERT(crp->crp_iv_start + csp->csp_ivlen <= crp->crp_ilen, + ("IV outside input length")); + } + KASSERT(crp->crp_payload_start == 0 || + crp->crp_payload_start < crp->crp_ilen, + ("invalid payload start")); + KASSERT(crp->crp_payload_start + crp->crp_payload_length <= + crp->crp_ilen, ("payload outside input length")); + if (csp->csp_mode == CSP_MODE_DIGEST || + csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) { + KASSERT(crp->crp_digest_start == 0 || + crp->crp_digest_start < crp->crp_ilen, + ("invalid digest start")); + /* XXX: For the mlen == 0 case this check isn't perfect. */ + KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= + crp->crp_ilen, + ("digest outside input length")); + } else { + KASSERT(crp->crp_digest_start == 0, + ("non-zero digest start for request without a digest")); + } + if (csp->csp_cipher_klen != 0) + KASSERT(csp->csp_cipher_key != NULL || + crp->crp_cipher_key != NULL, + ("cipher request without a key")); + if (csp->csp_auth_klen != 0) + KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL, + ("auth request without a key")); + KASSERT(crp->crp_callback != NULL, ("incoming crp without callback")); +} +#endif + /* * Add a crypto request to a queue, to be processed by the kernel thread. */ @@ -958,9 +1357,14 @@ int crypto_dispatch(struct cryptop *crp) { struct cryptocap *cap; - u_int32_t hid; int result; +#ifdef INVARIANTS + crp_sanity(crp); +#endif + + /* TODO: Handle CRYPTO_F_IV_GENERATE so drivers don't have to. */ + cryptostats.cs_ops++; #ifdef CRYPTO_TIMING @@ -987,16 +1391,12 @@ crypto_dispatch(struct cryptop *crp) } if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { - hid = crypto_ses2hid(crp->crp_session); - /* * Caller marked the request to be processed * immediately; dispatch it directly to the * driver unless the driver is currently blocked. */ - cap = crypto_checkdriver(hid); - /* Driver cannot disappeared when there is an active session. */ - KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); + cap = crp->crp_session->cap; if (!cap->cc_qblocked) { result = crypto_invoke(cap, crp, 0); if (result != ERESTART) @@ -1033,7 +1433,8 @@ crypto_kdispatch(struct cryptkop *krp) cryptostats.cs_kops++; - error = crypto_kinvoke(krp, krp->krp_crid); + krp->krp_cap = NULL; + error = crypto_kinvoke(krp); if (error == ERESTART) { CRYPTO_Q_LOCK(); TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); @@ -1081,15 +1482,14 @@ crypto_select_kdriver(const struct cryptkop *krp, int flags) match = CRYPTOCAP_F_SOFTWARE; best = NULL; again: - for (hid = 0; hid < crypto_drivers_num; hid++) { - cap = &crypto_drivers[hid]; + for (hid = 0; hid < crypto_drivers_size; hid++) { /* - * If it's not initialized, is in the process of - * going away, or is not appropriate (hardware - * or software based on match), then skip. + * If there is no driver for this slot, or the driver + * is not appropriate (hardware or software based on + * match), then skip. */ + cap = crypto_drivers[hid]; if (cap->cc_dev == NULL || - (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || (cap->cc_flags & match) == 0) continue; @@ -1111,30 +1511,33 @@ again: } /* - * Dispatch an asymmetric crypto request. + * Choose a driver for an asymmetric crypto request. */ -static int -crypto_kinvoke(struct cryptkop *krp, int crid) +static struct cryptocap * +crypto_lookup_kdriver(struct cryptkop *krp) { - struct cryptocap *cap = NULL; - int error; + struct cryptocap *cap; + uint32_t crid; - KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); - KASSERT(krp->krp_callback != NULL, - ("%s: krp->crp_callback == NULL", __func__)); + /* If this request is requeued, it might already have a driver. */ + cap = krp->krp_cap; + if (cap != NULL) + return (cap); - CRYPTO_DRIVER_LOCK(); + /* Use krp_crid to choose a driver. */ + crid = krp->krp_crid; if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { cap = crypto_checkdriver(crid); if (cap != NULL) { /* - * Driver present, it must support the necessary - * algorithm and, if s/w drivers are excluded, - * it must be registered as hardware-backed. + * Driver present, it must support the + * necessary algorithm and, if s/w drivers are + * excluded, it must be registered as + * hardware-backed. */ if (!kdriver_suitable(cap, krp) || (!crypto_devallowsoft && - (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) + (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) cap = NULL; } } else { @@ -1145,32 +1548,61 @@ crypto_kinvoke(struct cryptkop *krp, int crid) crid &= ~CRYPTOCAP_F_SOFTWARE; cap = crypto_select_kdriver(krp, crid); } - if (cap != NULL && !cap->cc_kqblocked) { - krp->krp_hid = cap - crypto_drivers; - cap->cc_koperations++; + + if (cap != NULL) { + krp->krp_cap = cap_ref(cap); + krp->krp_hid = cap->cc_hid; + } + return (cap); +} + +/* + * Dispatch an asymmetric crypto request. + */ +static int +crypto_kinvoke(struct cryptkop *krp) +{ + struct cryptocap *cap = NULL; + int error; + + KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); + KASSERT(krp->krp_callback != NULL, + ("%s: krp->crp_callback == NULL", __func__)); + + CRYPTO_DRIVER_LOCK(); + cap = crypto_lookup_kdriver(krp); + if (cap == NULL) { CRYPTO_DRIVER_UNLOCK(); - error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); - CRYPTO_DRIVER_LOCK(); - if (error == ERESTART) { - cap->cc_koperations--; - CRYPTO_DRIVER_UNLOCK(); - return (error); - } - } else { + krp->krp_status = ENODEV; + crypto_kdone(krp); + return (0); + } + + /* + * If the device is blocked, return ERESTART to requeue it. + */ + if (cap->cc_kqblocked) { /* - * NB: cap is !NULL if device is blocked; in - * that case return ERESTART so the operation - * is resubmitted if possible. + * XXX: Previously this set krp_status to ERESTART and + * invoked crypto_kdone but the caller would still + * requeue it. */ - error = (cap == NULL) ? ENODEV : ERESTART; + CRYPTO_DRIVER_UNLOCK(); + return (ERESTART); } - CRYPTO_DRIVER_UNLOCK(); - if (error) { - krp->krp_status = error; - crypto_kdone(krp); + cap->cc_koperations++; + CRYPTO_DRIVER_UNLOCK(); + error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); + if (error == ERESTART) { + CRYPTO_DRIVER_LOCK(); + cap->cc_koperations--; + CRYPTO_DRIVER_UNLOCK(); + return (error); } - return 0; + + KASSERT(error == 0, ("error %d returned from crypto_kprocess", error)); + return (0); } #ifdef CRYPTO_TIMING @@ -1204,13 +1636,10 @@ crypto_task_invoke(void *ctx, int pending) { struct cryptocap *cap; struct cryptop *crp; - int hid, result; + int result; crp = (struct cryptop *)ctx; - - hid = crypto_ses2hid(crp->crp_session); - cap = crypto_checkdriver(hid); - + cap = crp->crp_session->cap; result = crypto_invoke(cap, crp, 0); if (result == ERESTART) crypto_batch_enqueue(crp); @@ -1226,14 +1655,15 @@ crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); KASSERT(crp->crp_callback != NULL, ("%s: crp->crp_callback == NULL", __func__)); - KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); + KASSERT(crp->crp_session != NULL, + ("%s: crp->crp_session == NULL", __func__)); #ifdef CRYPTO_TIMING if (crypto_timing) crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); #endif if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { - struct cryptodesc *crd; + struct crypto_session_params csp; crypto_session_t nses; /* @@ -1242,14 +1672,32 @@ crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) * * XXX: What if there are more already queued requests for this * session? + * + * XXX: Real solution is to make sessions refcounted + * and force callers to hold a reference when + * assigning to crp_session. Could maybe change + * crypto_getreq to accept a session pointer to make + * that work. Alternatively, we could abandon the + * notion of rewriting crp_session in requests forcing + * the caller to deal with allocating a new session. + * Perhaps provide a method to allow a crp's session to + * be swapped that callers could use. */ + csp = crp->crp_session->csp; crypto_freesession(crp->crp_session); - for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) - crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); - - /* XXX propagate flags from initial session? */ - if (crypto_newsession(&nses, &(crp->crp_desc->CRD_INI), + /* + * XXX: Key pointers may no longer be valid. If we + * really want to support this we need to define the + * KPI such that 'csp' is required to be valid for the + * duration of a session by the caller perhaps. + * + * XXX: If the keys have been changed this will reuse + * the old keys. This probably suggests making + * rekeying more explicit and updating the key + * pointers in 'csp' when the keys change. + */ + if (crypto_newsession(&nses, &csp, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) crp->crp_session = nses; @@ -1264,13 +1712,9 @@ crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) } } -/* - * Release a set of crypto descriptors. - */ void crypto_freereq(struct cryptop *crp) { - struct cryptodesc *crd; if (crp == NULL) return; @@ -1300,36 +1744,18 @@ crypto_freereq(struct cryptop *crp) } #endif - while ((crd = crp->crp_desc) != NULL) { - crp->crp_desc = crd->crd_next; - uma_zfree(cryptodesc_zone, crd); - } uma_zfree(cryptop_zone, crp); } -/* - * Acquire a set of crypto descriptors. - */ struct cryptop * -crypto_getreq(int num) +crypto_getreq(crypto_session_t cses, int how) { - struct cryptodesc *crd; struct cryptop *crp; - crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); - if (crp != NULL) { - while (num--) { - crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); - if (crd == NULL) { - crypto_freereq(crp); - return NULL; - } - - crd->crd_next = crp->crp_desc; - crp->crp_desc = crd; - } - } - return crp; + MPASS(how == M_WAITOK || how == M_NOWAIT); + crp = uma_zalloc(cryptop_zone, how | M_ZERO); + crp->crp_session = cses; + return (crp); } /* @@ -1432,15 +1858,14 @@ crypto_kdone(struct cryptkop *krp) if (krp->krp_status != 0) cryptostats.cs_kerrs++; CRYPTO_DRIVER_LOCK(); - /* XXX: What if driver is loaded in the meantime? */ - if (krp->krp_hid < crypto_drivers_num) { - cap = &crypto_drivers[krp->krp_hid]; - KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); - cap->cc_koperations--; - if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) - crypto_remove(cap); - } + cap = krp->krp_cap; + KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); + cap->cc_koperations--; + if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP) + wakeup(cap); CRYPTO_DRIVER_UNLOCK(); + krp->krp_cap = NULL; + cap_rele(cap); ret_worker = CRYPTO_RETW(0); @@ -1457,11 +1882,12 @@ crypto_getfeat(int *featp) int hid, kalg, feat = 0; CRYPTO_DRIVER_LOCK(); - for (hid = 0; hid < crypto_drivers_num; hid++) { - const struct cryptocap *cap = &crypto_drivers[hid]; + for (hid = 0; hid < crypto_drivers_size; hid++) { + const struct cryptocap *cap = crypto_drivers[hid]; - if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && - !crypto_devallowsoft) { + if (cap == NULL || + ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && + !crypto_devallowsoft)) { continue; } for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) @@ -1500,7 +1926,6 @@ crypto_proc(void) struct cryptop *crp, *submit; struct cryptkop *krp; struct cryptocap *cap; - u_int32_t hid; int result, hint; #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) @@ -1517,15 +1942,14 @@ crypto_proc(void) submit = NULL; hint = 0; TAILQ_FOREACH(crp, &crp_q, crp_next) { - hid = crypto_ses2hid(crp->crp_session); - cap = crypto_checkdriver(hid); + cap = crp->crp_session->cap; /* * Driver cannot disappeared when there is an active * session. */ KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); - if (cap == NULL || cap->cc_dev == NULL) { + if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { /* Op needs to be migrated, process it. */ if (submit == NULL) submit = crp; @@ -1541,7 +1965,7 @@ crypto_proc(void) * better to just use a per-driver * queue instead. */ - if (crypto_ses2hid(submit->crp_session) == hid) + if (submit->crp_session->cap == cap) hint = CRYPTO_HINT_MORE; break; } else { @@ -1554,11 +1978,12 @@ crypto_proc(void) } if (submit != NULL) { TAILQ_REMOVE(&crp_q, submit, crp_next); - hid = crypto_ses2hid(submit->crp_session); - cap = crypto_checkdriver(hid); + cap = submit->crp_session->cap; KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); + CRYPTO_Q_UNLOCK(); result = crypto_invoke(cap, submit, hint); + CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the @@ -1569,8 +1994,7 @@ crypto_proc(void) * at the front. This should be ok; putting * it at the end does not work. */ - /* XXX validate sid again? */ - crypto_drivers[crypto_ses2hid(submit->crp_session)].cc_qblocked = 1; + cap->cc_qblocked = 1; TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); cryptostats.cs_blocks++; } @@ -1578,19 +2002,15 @@ crypto_proc(void) /* As above, but for key ops */ TAILQ_FOREACH(krp, &crp_kq, krp_next) { - cap = crypto_checkdriver(krp->krp_hid); - if (cap == NULL || cap->cc_dev == NULL) { + cap = krp->krp_cap; + if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { /* - * Operation needs to be migrated, invalidate - * the assigned device so it will reselect a - * new one below. Propagate the original - * crid selection flags if supplied. + * Operation needs to be migrated, + * clear krp_cap so a new driver is + * selected. */ - krp->krp_hid = krp->krp_crid & - (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); - if (krp->krp_hid == 0) - krp->krp_hid = - CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; + krp->krp_cap = NULL; + cap_rele(cap); break; } if (!cap->cc_kqblocked) @@ -1598,7 +2018,9 @@ crypto_proc(void) } if (krp != NULL) { TAILQ_REMOVE(&crp_kq, krp, krp_next); - result = crypto_kinvoke(krp, krp->krp_hid); + CRYPTO_Q_UNLOCK(); + result = crypto_kinvoke(krp); + CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the @@ -1609,8 +2031,7 @@ crypto_proc(void) * at the front. This should be ok; putting * it at the end does not work. */ - /* XXX validate sid again? */ - crypto_drivers[krp->krp_hid].cc_kqblocked = 1; + krp->krp_cap->cc_kqblocked = 1; TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); cryptostats.cs_kblocks++; } @@ -1731,9 +2152,9 @@ db_show_drivers(void) , "QB" , "KB" ); - for (hid = 0; hid < crypto_drivers_num; hid++) { - const struct cryptocap *cap = &crypto_drivers[hid]; - if (cap->cc_dev == NULL) + for (hid = 0; hid < crypto_drivers_size; hid++) { + const struct cryptocap *cap = crypto_drivers[hid]; + if (cap == NULL) continue; db_printf("%-12s %4u %4u %08x %2u %2u\n" , device_get_nameunit(cap->cc_dev) @@ -1756,15 +2177,15 @@ DB_SHOW_COMMAND(crypto, db_show_crypto) db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", - "Desc", "Callback"); + "Device", "Callback"); TAILQ_FOREACH(crp, &crp_q, crp_next) { db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" - , (int) crypto_ses2hid(crp->crp_session) + , crp->crp_session->cap->cc_hid , (int) crypto_ses2caps(crp->crp_session) , crp->crp_ilen, crp->crp_olen , crp->crp_etype , crp->crp_flags - , crp->crp_desc + , device_get_nameunit(crp->crp_session->cap->cc_dev) , crp->crp_callback ); } @@ -1775,7 +2196,7 @@ DB_SHOW_COMMAND(crypto, db_show_crypto) TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) { db_printf("%8td %4u %4u %04x %8p\n" , CRYPTO_RETW_ID(ret_worker) - , (int) crypto_ses2hid(crp->crp_session) + , crp->crp_session->cap->cc_hid , crp->crp_etype , crp->crp_flags , crp->crp_callback diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c index e43f1a9dff43..cf213f54d947 100644 --- a/sys/opencrypto/cryptodev.c +++ b/sys/opencrypto/cryptodev.c @@ -51,7 +51,6 @@ __FBSDID("$FreeBSD$"); #include <sys/file.h> #include <sys/filedesc.h> #include <sys/errno.h> -#include <sys/uio.h> #include <sys/random.h> #include <sys/conf.h> #include <sys/kernel.h> @@ -270,23 +269,19 @@ struct csession { u_int32_t ses; struct mtx lock; /* for op submission */ - u_int32_t cipher; struct enc_xform *txform; - u_int32_t mac; - struct auth_hash *thash; - - caddr_t key; - int keylen; + int hashsize; + int ivsize; + int mode; - caddr_t mackey; - int mackeylen; + void *key; + void *mackey; }; struct cryptop_data { struct csession *cse; - struct iovec iovec[1]; - struct uio uio; + char *buf; bool done; }; @@ -326,9 +321,9 @@ static struct fileops cryptofops = { static struct csession *csefind(struct fcrypt *, u_int); static bool csedelete(struct fcrypt *, u_int); -static struct csession *csecreate(struct fcrypt *, crypto_session_t, caddr_t, - u_int64_t, caddr_t, u_int64_t, u_int32_t, u_int32_t, struct enc_xform *, - struct auth_hash *); +static struct csession *csecreate(struct fcrypt *, crypto_session_t, + struct crypto_session_params *, struct enc_xform *, void *, + struct auth_hash *, void *); static void csefree(struct csession *); static int cryptodev_op(struct csession *, struct crypt_op *, @@ -375,7 +370,7 @@ cryptof_ioctl( struct thread *td) { #define SES2(p) ((struct session2_op *)p) - struct cryptoini cria, crie; + struct crypto_session_params csp; struct fcrypt *fcr = fp->f_data; struct csession *cse; struct session_op *sop; @@ -383,6 +378,8 @@ cryptof_ioctl( struct crypt_aead *caead; struct enc_xform *txform = NULL; struct auth_hash *thash = NULL; + void *key = NULL; + void *mackey = NULL; struct crypt_kop *kop; crypto_session_t cses; u_int32_t ses; @@ -487,18 +484,37 @@ cryptof_ioctl( case CRYPTO_RIPEMD160_HMAC: thash = &auth_hash_hmac_ripemd_160; break; +#ifdef COMPAT_FREEBSD12 case CRYPTO_AES_128_NIST_GMAC: - thash = &auth_hash_nist_gmac_aes_128; - break; case CRYPTO_AES_192_NIST_GMAC: - thash = &auth_hash_nist_gmac_aes_192; - break; case CRYPTO_AES_256_NIST_GMAC: - thash = &auth_hash_nist_gmac_aes_256; + /* Should always be paired with GCM. */ + if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { + CRYPTDEB("GMAC without GCM"); + return (EINVAL); + } + break; +#endif + case CRYPTO_AES_NIST_GMAC: + switch (sop->mackeylen * 8) { + case 128: + thash = &auth_hash_nist_gmac_aes_128; + break; + case 192: + thash = &auth_hash_nist_gmac_aes_192; + break; + case 256: + thash = &auth_hash_nist_gmac_aes_256; + break; + default: + CRYPTDEB("invalid GMAC key length"); + SDT_PROBE1(opencrypto, dev, ioctl, error, + __LINE__); + return (EINVAL); + } break; - case CRYPTO_AES_CCM_CBC_MAC: - switch (sop->keylen) { + switch (sop->mackeylen) { case 16: thash = &auth_hash_ccm_cbc_mac_128; break; @@ -554,12 +570,52 @@ cryptof_ioctl( return (EINVAL); } - bzero(&crie, sizeof(crie)); - bzero(&cria, sizeof(cria)); + if (txform == NULL && thash == NULL) + return (EINVAL); + + memset(&csp, 0, sizeof(csp)); + + if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { + switch (sop->mac) { +#ifdef COMPAT_FREEBSD12 + case CRYPTO_AES_128_NIST_GMAC: + case CRYPTO_AES_192_NIST_GMAC: + case CRYPTO_AES_256_NIST_GMAC: + if (sop->keylen != sop->mackeylen) + return (EINVAL); + break; +#endif + case 0: + break; + default: + return (EINVAL); + } + csp.csp_mode = CSP_MODE_AEAD; + } else if (sop->cipher == CRYPTO_AES_CCM_16) { + switch (sop->mac) { +#ifdef COMPAT_FREEBSD12 + case CRYPTO_AES_CCM_CBC_MAC: + if (sop->keylen != sop->mackeylen) + return (EINVAL); + thash = NULL; + break; +#endif + case 0: + break; + default: + return (EINVAL); + } + csp.csp_mode = CSP_MODE_AEAD; + } else if (txform && thash) + csp.csp_mode = CSP_MODE_ETA; + else if (txform) + csp.csp_mode = CSP_MODE_CIPHER; + else + csp.csp_mode = CSP_MODE_DIGEST; if (txform) { - crie.cri_alg = txform->type; - crie.cri_klen = sop->keylen * 8; + csp.csp_cipher_alg = txform->type; + csp.csp_cipher_klen = sop->keylen; if (sop->keylen > txform->maxkey || sop->keylen < txform->minkey) { CRYPTDEB("invalid cipher parameters"); @@ -569,22 +625,21 @@ cryptof_ioctl( goto bail; } - crie.cri_key = malloc(crie.cri_klen / 8, - M_XDATA, M_WAITOK); - if ((error = copyin(sop->key, crie.cri_key, - crie.cri_klen / 8))) { + key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); + error = copyin(sop->key, key, csp.csp_cipher_klen); + if (error) { CRYPTDEB("invalid key"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } - if (thash) - crie.cri_next = &cria; + csp.csp_cipher_key = key; + csp.csp_ivlen = txform->ivsize; } if (thash) { - cria.cri_alg = thash->type; - cria.cri_klen = sop->mackeylen * 8; + csp.csp_auth_alg = thash->type; + csp.csp_auth_klen = sop->mackeylen; if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { CRYPTDEB("invalid mac key length"); @@ -594,17 +649,24 @@ cryptof_ioctl( goto bail; } - if (cria.cri_klen) { - cria.cri_key = malloc(cria.cri_klen / 8, - M_XDATA, M_WAITOK); - if ((error = copyin(sop->mackey, cria.cri_key, - cria.cri_klen / 8))) { + if (csp.csp_auth_klen) { + mackey = malloc(csp.csp_auth_klen, M_XDATA, + M_WAITOK); + error = copyin(sop->mackey, mackey, + csp.csp_auth_klen); + if (error) { CRYPTDEB("invalid mac key"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } + csp.csp_auth_key = mackey; } + + if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) + csp.csp_ivlen = AES_GCM_IV_LEN; + if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) + csp.csp_ivlen = AES_CCM_IV_LEN; } /* NB: CIOCGSESSION2 has the crid */ @@ -623,16 +685,14 @@ cryptof_ioctl( } } else crid = CRYPTOCAP_F_HARDWARE; - error = crypto_newsession(&cses, (txform ? &crie : &cria), crid); + error = crypto_newsession(&cses, &csp, crid); if (error) { CRYPTDEB("crypto_newsession"); SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } - cse = csecreate(fcr, cses, crie.cri_key, crie.cri_klen, - cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform, - thash); + cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey); if (cse == NULL) { crypto_freesession(cses); @@ -652,10 +712,8 @@ cryptof_ioctl( } bail: if (error) { - if (crie.cri_key) - free(crie.cri_key, M_XDATA); - if (cria.cri_key) - free(cria.cri_key, M_XDATA); + free(key, M_XDATA); + free(mackey, M_XDATA); } #ifdef COMPAT_FREEBSD32 else { @@ -773,20 +831,11 @@ static struct cryptop_data * cod_alloc(struct csession *cse, size_t len, struct thread *td) { struct cryptop_data *cod; - struct uio *uio; cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); cod->cse = cse; - uio = &cod->uio; - uio->uio_iov = cod->iovec; - uio->uio_iovcnt = 1; - uio->uio_resid = len; - uio->uio_segflg = UIO_SYSSPACE; - uio->uio_rw = UIO_WRITE; - uio->uio_td = td; - uio->uio_iov[0].iov_len = len; - uio->uio_iov[0].iov_base = malloc(len, M_XDATA, M_WAITOK); + cod->buf = malloc(len, M_XDATA, M_WAITOK); return (cod); } @@ -794,7 +843,7 @@ static void cod_free(struct cryptop_data *cod) { - free(cod->uio.uio_iov[0].iov_base, M_XDATA); + free(cod->buf, M_XDATA); free(cod, M_XDATA); } @@ -803,8 +852,10 @@ cryptodev_warn(struct csession *cse) { static struct timeval arc4warn, blfwarn, castwarn, deswarn, md5warn; static struct timeval skipwarn, tdeswarn; + const struct crypto_session_params *csp; - switch (cse->cipher) { + csp = crypto_get_params(cse->cses); + switch (csp->csp_cipher_alg) { case CRYPTO_DES_CBC: if (ratecheck(&deswarn, &warninterval)) gone_in(13, "DES cipher via /dev/crypto"); @@ -831,7 +882,7 @@ cryptodev_warn(struct csession *cse) break; } - switch (cse->mac) { + switch (csp->csp_auth_alg) { case CRYPTO_MD5_HMAC: if (ratecheck(&md5warn, &warninterval)) gone_in(13, "MD5-HMAC authenticator via /dev/crypto"); @@ -848,7 +899,6 @@ cryptodev_op( { struct cryptop_data *cod = NULL; struct cryptop *crp = NULL; - struct cryptodesc *crde = NULL, *crda = NULL; int error; if (cop->len > 256*1024-4) { @@ -863,106 +913,135 @@ cryptodev_op( } } - if (cse->thash) - cod = cod_alloc(cse, cop->len + cse->thash->hashsize, td); - else - cod = cod_alloc(cse, cop->len, td); - - crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL)); - if (crp == NULL) { + if (cop->mac && cse->hashsize == 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - error = ENOMEM; + error = EINVAL; goto bail; } - if (cse->thash && cse->txform) { - if (cop->flags & COP_F_CIPHER_FIRST) { - crde = crp->crp_desc; - crda = crde->crd_next; - } else { - crda = crp->crp_desc; - crde = crda->crd_next; + /* + * The COP_F_CIPHER_FIRST flag predates explicit session + * modes, but the only way it was used was for EtA so allow it + * as long as it is consistent with EtA. + */ + if (cop->flags & COP_F_CIPHER_FIRST) { + if (cop->op != COP_ENCRYPT) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + return (EINVAL); } - } else if (cse->thash) { - crda = crp->crp_desc; - } else if (cse->txform) { - crde = crp->crp_desc; - } else { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - error = EINVAL; - goto bail; } - if ((error = copyin(cop->src, cod->uio.uio_iov[0].iov_base, - cop->len))) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - goto bail; - } + cod = cod_alloc(cse, cop->len + cse->hashsize, td); - if (crda) { - crda->crd_skip = 0; - crda->crd_len = cop->len; - crda->crd_inject = cop->len; + crp = crypto_getreq(cse->cses, M_WAITOK); - crda->crd_alg = cse->mac; - crda->crd_key = cse->mackey; - crda->crd_klen = cse->mackeylen * 8; + error = copyin(cop->src, cod->buf, cop->len); + if (error) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + goto bail; } + crp->crp_payload_start = 0; + crp->crp_payload_length = cop->len; + if (cse->hashsize) + crp->crp_digest_start = cop->len; - if (crde) { - if (cop->op == COP_ENCRYPT) - crde->crd_flags |= CRD_F_ENCRYPT; - else - crde->crd_flags &= ~CRD_F_ENCRYPT; - crde->crd_len = cop->len; - crde->crd_inject = 0; - - crde->crd_alg = cse->cipher; - crde->crd_key = cse->key; - crde->crd_klen = cse->keylen * 8; + switch (cse->mode) { + case CSP_MODE_COMPRESS: + switch (cop->op) { + case COP_ENCRYPT: + crp->crp_op = CRYPTO_OP_COMPRESS; + break; + case COP_DECRYPT: + crp->crp_op = CRYPTO_OP_DECOMPRESS; + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; + } + break; + case CSP_MODE_CIPHER: + switch (cop->op) { + case COP_ENCRYPT: + crp->crp_op = CRYPTO_OP_ENCRYPT; + break; + case COP_DECRYPT: + crp->crp_op = CRYPTO_OP_DECRYPT; + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; + } + break; + case CSP_MODE_DIGEST: + switch (cop->op) { + case 0: + case COP_ENCRYPT: + case COP_DECRYPT: + crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; + } + break; + case CSP_MODE_ETA: + switch (cop->op) { + case COP_ENCRYPT: + crp->crp_op = CRYPTO_OP_ENCRYPT | + CRYPTO_OP_COMPUTE_DIGEST; + break; + case COP_DECRYPT: + crp->crp_op = CRYPTO_OP_DECRYPT | + CRYPTO_OP_VERIFY_DIGEST; + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; + } + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; } - crp->crp_ilen = cop->len; - crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM - | (cop->flags & COP_F_BATCH); - crp->crp_uio = &cod->uio; + crp->crp_ilen = cop->len + cse->hashsize; + crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); + crp->crp_buf = cod->buf; + crp->crp_buf_type = CRYPTO_BUF_CONTIG; crp->crp_callback = cryptodev_cb; - crp->crp_session = cse->cses; crp->crp_opaque = cod; if (cop->iv) { - if (crde == NULL) { + if (cse->ivsize == 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } - if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ + error = copyin(cop->iv, crp->crp_iv, cse->ivsize); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - error = EINVAL; goto bail; } - if ((error = copyin(cop->iv, crde->crd_iv, - cse->txform->ivsize))) { + crp->crp_flags |= CRYPTO_F_IV_SEPARATE; + } else if (cse->ivsize != 0) { + crp->crp_iv_start = 0; + crp->crp_payload_start += cse->ivsize; + crp->crp_payload_length -= cse->ivsize; + } + + if (cop->mac != NULL) { + error = copyin(cop->mac, cod->buf + cop->len, cse->hashsize); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } - crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; - crde->crd_skip = 0; - } else if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ - crde->crd_skip = 0; - } else if (crde) { - crde->crd_flags |= CRD_F_IV_PRESENT; - crde->crd_skip = cse->txform->ivsize; - crde->crd_len -= cse->txform->ivsize; - } - - if (cop->mac && crda == NULL) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - error = EINVAL; - goto bail; } cryptodev_warn(cse); - again: /* * Let the dispatch run unlocked, then, interlock against the @@ -995,18 +1074,20 @@ again: goto bail; } - if (cop->dst && - (error = copyout(cod->uio.uio_iov[0].iov_base, cop->dst, - cop->len))) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - goto bail; + if (cop->dst != NULL) { + error = copyout(cod->buf, cop->dst, cop->len); + if (error) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + goto bail; + } } - if (cop->mac && - (error = copyout((caddr_t)cod->uio.uio_iov[0].iov_base + cop->len, - cop->mac, cse->thash->hashsize))) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - goto bail; + if (cop->mac != NULL) { + error = copyout(cod->buf + cop->len, cop->mac, cse->hashsize); + if (error) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + goto bail; + } } bail: @@ -1027,7 +1108,6 @@ cryptodev_aead( { struct cryptop_data *cod = NULL; struct cryptop *crp = NULL; - struct cryptodesc *crde = NULL, *crda = NULL; int error; if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { @@ -1035,99 +1115,122 @@ cryptodev_aead( return (E2BIG); } - if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL || + if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || (caead->len % cse->txform->blocksize) != 0) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); } - cod = cod_alloc(cse, caead->aadlen + caead->len + cse->thash->hashsize, - td); + /* + * The COP_F_CIPHER_FIRST flag predates explicit session + * modes, but the only way it was used was for EtA so allow it + * as long as it is consistent with EtA. + */ + if (caead->flags & COP_F_CIPHER_FIRST) { + if (caead->op != COP_ENCRYPT) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + return (EINVAL); + } + } + + cod = cod_alloc(cse, caead->aadlen + caead->len + cse->hashsize, td); + + crp = crypto_getreq(cse->cses, M_WAITOK); - crp = crypto_getreq(2); - if (crp == NULL) { - error = ENOMEM; + error = copyin(caead->aad, cod->buf, caead->aadlen); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } + crp->crp_aad_start = 0; + crp->crp_aad_length = caead->aadlen; - if (caead->flags & COP_F_CIPHER_FIRST) { - crde = crp->crp_desc; - crda = crde->crd_next; - } else { - crda = crp->crp_desc; - crde = crda->crd_next; - } - - if ((error = copyin(caead->aad, cod->uio.uio_iov[0].iov_base, - caead->aadlen))) { + error = copyin(caead->src, cod->buf + caead->aadlen, caead->len); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } + crp->crp_payload_start = caead->aadlen; + crp->crp_payload_length = caead->len; + crp->crp_digest_start = caead->aadlen + caead->len; - if ((error = copyin(caead->src, (char *)cod->uio.uio_iov[0].iov_base + - caead->aadlen, caead->len))) { + switch (cse->mode) { + case CSP_MODE_AEAD: + switch (caead->op) { + case COP_ENCRYPT: + crp->crp_op = CRYPTO_OP_ENCRYPT | + CRYPTO_OP_COMPUTE_DIGEST; + break; + case COP_DECRYPT: + crp->crp_op = CRYPTO_OP_DECRYPT | + CRYPTO_OP_VERIFY_DIGEST; + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; + } + break; + case CSP_MODE_ETA: + switch (caead->op) { + case COP_ENCRYPT: + crp->crp_op = CRYPTO_OP_ENCRYPT | + CRYPTO_OP_COMPUTE_DIGEST; + break; + case COP_DECRYPT: + crp->crp_op = CRYPTO_OP_DECRYPT | + CRYPTO_OP_VERIFY_DIGEST; + break; + default: + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; + goto bail; + } + break; + default: SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + error = EINVAL; goto bail; } - /* - * For GCM/CCM, crd_len covers only the AAD. For other ciphers - * chained with an HMAC, crd_len covers both the AAD and the - * cipher text. - */ - crda->crd_skip = 0; - if (cse->cipher == CRYPTO_AES_NIST_GCM_16 || - cse->cipher == CRYPTO_AES_CCM_16) - crda->crd_len = caead->aadlen; - else - crda->crd_len = caead->aadlen + caead->len; - crda->crd_inject = caead->aadlen + caead->len; - - crda->crd_alg = cse->mac; - crda->crd_key = cse->mackey; - crda->crd_klen = cse->mackeylen * 8; - - if (caead->op == COP_ENCRYPT) - crde->crd_flags |= CRD_F_ENCRYPT; - else - crde->crd_flags &= ~CRD_F_ENCRYPT; - crde->crd_skip = caead->aadlen; - crde->crd_len = caead->len; - crde->crd_inject = caead->aadlen; - - crde->crd_alg = cse->cipher; - crde->crd_key = cse->key; - crde->crd_klen = cse->keylen * 8; - - crp->crp_ilen = caead->aadlen + caead->len; - crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM - | (caead->flags & COP_F_BATCH); - crp->crp_uio = &cod->uio; + crp->crp_ilen = caead->aadlen + caead->len + cse->hashsize; + crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); + crp->crp_buf = cod->buf; + crp->crp_buf_type = CRYPTO_BUF_CONTIG; crp->crp_callback = cryptodev_cb; - crp->crp_session = cse->cses; crp->crp_opaque = cod; if (caead->iv) { - if (caead->ivlen > sizeof(crde->crd_iv)) { + /* + * Permit a 16-byte IV for AES-XTS, but only use the + * first 8 bytes as a block number. + */ + if (cse->mode == CSP_MODE_ETA && + caead->ivlen == AES_BLOCK_LEN && + cse->ivsize == AES_XTS_IV_LEN) + caead->ivlen = AES_XTS_IV_LEN; + + if (caead->ivlen != cse->ivsize) { error = EINVAL; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } - if ((error = copyin(caead->iv, crde->crd_iv, caead->ivlen))) { + error = copyin(caead->iv, crp->crp_iv, cse->ivsize); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } - crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; + crp->crp_flags |= CRYPTO_F_IV_SEPARATE; } else { - crde->crd_flags |= CRD_F_IV_PRESENT; - crde->crd_skip += cse->txform->ivsize; - crde->crd_len -= cse->txform->ivsize; + crp->crp_iv_start = crp->crp_payload_start; + crp->crp_payload_start += cse->ivsize; + crp->crp_payload_length -= cse->ivsize; } - if ((error = copyin(caead->tag, (caddr_t)cod->uio.uio_iov[0].iov_base + - caead->len + caead->aadlen, cse->thash->hashsize))) { + error = copyin(caead->tag, cod->buf + caead->len + caead->aadlen, + cse->hashsize); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } @@ -1164,15 +1267,18 @@ again: goto bail; } - if (caead->dst && (error = copyout( - (caddr_t)cod->uio.uio_iov[0].iov_base + caead->aadlen, caead->dst, - caead->len))) { - SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); - goto bail; + if (caead->dst != NULL) { + error = copyout(cod->buf + caead->aadlen, caead->dst, + caead->len); + if (error) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); + goto bail; + } } - if ((error = copyout((caddr_t)cod->uio.uio_iov[0].iov_base + - caead->aadlen + caead->len, caead->tag, cse->thash->hashsize))) { + error = copyout(cod->buf + caead->aadlen + caead->len, caead->tag, + cse->hashsize); + if (error) { SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; } @@ -1202,13 +1308,11 @@ cryptodev_cb(struct cryptop *crp) return (0); } -static int -cryptodevkey_cb(void *op) +static void +cryptodevkey_cb(struct cryptkop *krp) { - struct cryptkop *krp = (struct cryptkop *) op; wakeup_one(krp); - return (0); } static int @@ -1267,7 +1371,7 @@ cryptodev_key(struct crypt_kop *kop) krp->krp_oparams = kop->crk_oparams; krp->krp_crid = kop->crk_crid; krp->krp_status = 0; - krp->krp_callback = (int (*) (struct cryptkop *)) cryptodevkey_cb; + krp->krp_callback = cryptodevkey_cb; for (i = 0; i < CRK_MAXPARAM; i++) { if (kop->crk_param[i].crp_nbits > 65536) { @@ -1303,7 +1407,7 @@ cryptodev_key(struct crypt_kop *kop) goto fail; } - kop->crk_crid = krp->krp_crid; /* device that did the work */ + kop->crk_crid = krp->krp_hid; /* device that did the work */ if (krp->krp_status != 0) { error = krp->krp_status; SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); @@ -1429,9 +1533,9 @@ csedelete(struct fcrypt *fcr, u_int ses) } struct csession * -csecreate(struct fcrypt *fcr, crypto_session_t cses, caddr_t key, u_int64_t keylen, - caddr_t mackey, u_int64_t mackeylen, u_int32_t cipher, u_int32_t mac, - struct enc_xform *txform, struct auth_hash *thash) +csecreate(struct fcrypt *fcr, crypto_session_t cses, + struct crypto_session_params *csp, struct enc_xform *txform, + void *key, struct auth_hash *thash, void *mackey) { struct csession *cse; @@ -1441,14 +1545,17 @@ csecreate(struct fcrypt *fcr, crypto_session_t cses, caddr_t key, u_int64_t keyl mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); refcount_init(&cse->refs, 1); cse->key = key; - cse->keylen = keylen/8; cse->mackey = mackey; - cse->mackeylen = mackeylen/8; + cse->mode = csp->csp_mode; cse->cses = cses; - cse->cipher = cipher; - cse->mac = mac; cse->txform = txform; - cse->thash = thash; + if (thash != NULL) + cse->hashsize = thash->hashsize; + else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) + cse->hashsize = AES_GMAC_HASH_LEN; + else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) + cse->hashsize = AES_CBC_MAC_HASH_LEN; + cse->ivsize = csp->csp_ivlen; mtx_lock(&fcr->lock); TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); cse->ses = fcr->sesn++; diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h index bd71e518c576..f9aa8a053c36 100644 --- a/sys/opencrypto/cryptodev.h +++ b/sys/opencrypto/cryptodev.h @@ -71,7 +71,6 @@ /* Some initial values */ #define CRYPTO_DRIVERS_INITIAL 4 -#define CRYPTO_SW_SESSIONS 32 /* Hash values */ #define NULL_HASH_LEN 16 @@ -189,11 +188,13 @@ #define CRYPTO_CAMELLIA_CBC 21 #define CRYPTO_AES_XTS 22 #define CRYPTO_AES_ICM 23 /* commonly known as CTR mode */ -#define CRYPTO_AES_NIST_GMAC 24 /* cipher side */ +#define CRYPTO_AES_NIST_GMAC 24 /* GMAC only */ #define CRYPTO_AES_NIST_GCM_16 25 /* 16 byte ICV */ +#ifdef _KERNEL #define CRYPTO_AES_128_NIST_GMAC 26 /* auth side */ #define CRYPTO_AES_192_NIST_GMAC 27 /* auth side */ #define CRYPTO_AES_256_NIST_GMAC 28 /* auth side */ +#endif #define CRYPTO_BLAKE2B 29 /* Blake2b hash */ #define CRYPTO_BLAKE2S 30 /* Blake2s hash */ #define CRYPTO_CHACHA20 31 /* Chacha20 stream cipher */ @@ -378,6 +379,13 @@ struct cryptostats { #ifdef _KERNEL +/* + * Return values for cryptodev_probesession methods. + */ +#define CRYPTODEV_PROBE_HARDWARE (-100) +#define CRYPTODEV_PROBE_ACCEL_SOFTWARE (-200) +#define CRYPTODEV_PROBE_SOFTWARE (-500) + #if 0 #define CRYPTDEB(s, ...) do { \ printf("%s:%d: " s "\n", __FILE__, __LINE__, ## __VA_ARGS__); \ @@ -386,40 +394,29 @@ struct cryptostats { #define CRYPTDEB(...) do { } while (0) #endif -/* Standard initialization structure beginning */ -struct cryptoini { - int cri_alg; /* Algorithm to use */ - int cri_klen; /* Key length, in bits */ - int cri_mlen; /* Number of bytes we want from the - entire hash. 0 means all. */ - caddr_t cri_key; /* key to use */ - u_int8_t cri_iv[EALG_MAX_BLOCK_LEN]; /* IV to use */ - struct cryptoini *cri_next; -}; +struct crypto_session_params { + int csp_mode; /* Type of operations to perform. */ + +#define CSP_MODE_NONE 0 +#define CSP_MODE_COMPRESS 1 /* Compression/decompression. */ +#define CSP_MODE_CIPHER 2 /* Encrypt/decrypt. */ +#define CSP_MODE_DIGEST 3 /* Compute/verify digest. */ +#define CSP_MODE_AEAD 4 /* Combined auth/encryption. */ +#define CSP_MODE_ETA 5 /* IPsec style encrypt-then-auth */ -/* Describe boundaries of a single crypto operation */ -struct cryptodesc { - int crd_skip; /* How many bytes to ignore from start */ - int crd_len; /* How many bytes to process */ - int crd_inject; /* Where to inject results, if applicable */ - int crd_flags; + int csp_flags; -#define CRD_F_ENCRYPT 0x01 /* Set when doing encryption */ -#define CRD_F_IV_PRESENT 0x02 /* When encrypting, IV is already in - place, so don't copy. */ -#define CRD_F_IV_EXPLICIT 0x04 /* IV explicitly provided */ -#define CRD_F_DSA_SHA_NEEDED 0x08 /* Compute SHA-1 of buffer for DSA */ -#define CRD_F_COMP 0x0f /* Set when doing compression */ -#define CRD_F_KEY_EXPLICIT 0x10 /* Key explicitly provided */ + int csp_ivlen; /* IV length in bytes. */ - struct cryptoini CRD_INI; /* Initialization/context data */ -#define crd_esn CRD_INI.cri_esn -#define crd_iv CRD_INI.cri_iv -#define crd_key CRD_INI.cri_key -#define crd_alg CRD_INI.cri_alg -#define crd_klen CRD_INI.cri_klen + int csp_cipher_alg; + int csp_cipher_klen; /* Key length in bytes. */ + const void *csp_cipher_key; - struct cryptodesc *crd_next; + int csp_auth_alg; + int csp_auth_klen; /* Key length in bytes. */ + const void *csp_auth_key; + int csp_auth_mlen; /* Number of digest bytes to use. + 0 means all. */ }; /* Structure describing complete operation */ @@ -444,8 +441,6 @@ struct cryptop { */ int crp_flags; -#define CRYPTO_F_IMBUF 0x0001 /* Input/output are mbuf chains */ -#define CRYPTO_F_IOV 0x0002 /* Input/output are uio */ #define CRYPTO_F_BATCH 0x0008 /* Batch op if possible */ #define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */ #define CRYPTO_F_DONE 0x0020 /* Operation completed */ @@ -458,14 +453,35 @@ struct cryptop { * order there are submitted. Applied only * if CRYPTO_F_ASYNC flags is set */ +#define CRYPTO_F_IV_SEPARATE 0x0200 /* Use crp_iv[] as IV. */ +#define CRYPTO_F_IV_GENERATE 0x0400 /* Generate a random IV and store. */ + + int crp_op; union { caddr_t crp_buf; /* Data to be processed */ struct mbuf *crp_mbuf; struct uio *crp_uio; }; - void * crp_opaque; /* Opaque pointer, passed along */ - struct cryptodesc *crp_desc; /* Linked list of processing descriptors */ + int crp_buf_type; /* Which union member describes data. */ + + int crp_aad_start; /* Location of AAD. */ + int crp_aad_length; /* 0 => no AAD. */ + int crp_iv_start; /* Location of IV. IV length is from + * the session. + */ + int crp_payload_start; /* Location of ciphertext. */ + int crp_payload_length; + int crp_digest_start; /* Location of MAC/tag. Length is + * from the session. + */ + + uint8_t crp_iv[EALG_MAX_BLOCK_LEN]; /* IV if IV_SEPARATE. */ + + const void *crp_cipher_key; /* New cipher key if non-NULL. */ + const void *crp_auth_key; /* New auth key if non-NULL. */ + + void *crp_opaque; /* Opaque pointer, passed along */ int (*crp_callback)(struct cryptop *); /* Callback function */ @@ -485,11 +501,18 @@ struct cryptop { (crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) #define CRYPTO_BUF_CONTIG 0x0 -#define CRYPTO_BUF_IOV 0x1 +#define CRYPTO_BUF_UIO 0x1 #define CRYPTO_BUF_MBUF 0x2 -#define CRYPTO_OP_DECRYPT 0x0 -#define CRYPTO_OP_ENCRYPT 0x1 +/* Flags in crp_op. */ +#define CRYPTO_OP_DECRYPT 0x0 +#define CRYPTO_OP_ENCRYPT 0x1 +#define CRYPTO_OP_IS_ENCRYPT(op) ((op) & CRYPTO_OP_ENCRYPT) +#define CRYPTO_OP_COMPUTE_DIGEST 0x0 +#define CRYPTO_OP_VERIFY_DIGEST 0x2 +#define CRYPTO_OP_DECOMPRESS CRYPTO_OP_DECRYPT +#define CRYPTO_OP_COMPRESS CRYPTO_OP_ENCRYPT +#define CRYPTO_OP_IS_COMPRESS(op) ((op) & CRYPTO_OP_COMPRESS) /* * Hints passed to process methods. @@ -504,18 +527,24 @@ struct cryptkop { u_short krp_iparams; /* # of input parameters */ u_short krp_oparams; /* # of output parameters */ u_int krp_crid; /* desired device, etc. */ - u_int32_t krp_hid; + uint32_t krp_hid; /* device used */ struct crparam krp_param[CRK_MAXPARAM]; /* kvm */ - int (*krp_callback)(struct cryptkop *); + void (*krp_callback)(struct cryptkop *); + struct cryptocap *krp_cap; }; uint32_t crypto_ses2hid(crypto_session_t crypto_session); uint32_t crypto_ses2caps(crypto_session_t crypto_session); void *crypto_get_driver_session(crypto_session_t crypto_session); +const struct crypto_session_params *crypto_get_params( + crypto_session_t crypto_session); +struct auth_hash *crypto_auth_hash(const struct crypto_session_params *csp); +struct enc_xform *crypto_cipher(const struct crypto_session_params *csp); MALLOC_DECLARE(M_CRYPTO_DATA); -extern int crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int hard); +extern int crypto_newsession(crypto_session_t *cses, + const struct crypto_session_params *params, int hard); extern void crypto_freesession(crypto_session_t cses); #define CRYPTOCAP_F_HARDWARE CRYPTO_FLAG_HARDWARE #define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE @@ -525,10 +554,7 @@ extern int32_t crypto_get_driverid(device_t dev, size_t session_size, extern int crypto_find_driver(const char *); extern device_t crypto_find_device_byhid(int hid); extern int crypto_getcaps(int hid); -extern int crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, - u_int32_t flags); extern int crypto_kregister(u_int32_t, int, u_int32_t); -extern int crypto_unregister(u_int32_t driverid, int alg); extern int crypto_unregister_all(u_int32_t driverid); extern int crypto_dispatch(struct cryptop *crp); extern int crypto_kdispatch(struct cryptkop *); @@ -540,17 +566,30 @@ extern void crypto_kdone(struct cryptkop *); extern int crypto_getfeat(int *); extern void crypto_freereq(struct cryptop *crp); -extern struct cryptop *crypto_getreq(int num); +extern struct cryptop *crypto_getreq(crypto_session_t cses, int how); extern int crypto_usercrypto; /* userland may do crypto requests */ extern int crypto_userasymcrypto; /* userland may do asym crypto reqs */ extern int crypto_devallowsoft; /* only use hardware crypto */ +/* Helper routines for drivers to initialize auth contexts for HMAC. */ +struct auth_hash; + +void hmac_init_ipad(struct auth_hash *axf, const char *key, int klen, + void *auth_ctx); +void hmac_init_opad(struct auth_hash *axf, const char *key, int klen, + void *auth_ctx); + /* * Crypto-related utility routines used mainly by drivers. * * XXX these don't really belong here; but for now they're * kept apart from the rest of the system. + * + * Similar to m_copyback/data, *_copyback copy data from the 'src' + * buffer into the crypto request's data buffer while *_copydata copy + * data from the crypto request's data buffer into the the 'dst' + * buffer. */ struct uio; extern void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp); @@ -564,14 +603,13 @@ struct iovec; extern int crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, int *cnt, int *allocated); -extern void crypto_copyback(int flags, caddr_t buf, int off, int size, - c_caddr_t in); -extern void crypto_copydata(int flags, caddr_t buf, int off, int size, - caddr_t out); -extern int crypto_apply(int flags, caddr_t buf, int off, int len, +void crypto_copyback(struct cryptop *crp, int off, int size, + const void *src); +void crypto_copydata(struct cryptop *crp, int off, int size, void *dst); +int crypto_apply(struct cryptop *crp, int off, int len, int (*f)(void *, void *, u_int), void *arg); - -extern void *crypto_contiguous_subsegment(int, void *, size_t, size_t); +void *crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, + size_t len); #endif /* _KERNEL */ #endif /* _CRYPTO_CRYPTO_H_ */ diff --git a/sys/opencrypto/cryptodev_if.m b/sys/opencrypto/cryptodev_if.m index 49caa5536c06..1d767be3f4e4 100644 --- a/sys/opencrypto/cryptodev_if.m +++ b/sys/opencrypto/cryptodev_if.m @@ -40,32 +40,138 @@ CODE { }; /** - * Crypto driver method to initialize a new session object with the given - * initialization parameters (cryptoini). The driver's session memory object - * is already allocated and zeroed, like driver softcs. It is accessed with + * @brief Probe to see if a crypto driver supports a session. + * + * The crypto framework invokes this method on each crypto driver when + * creating a session for symmetric crypto operations to determine if + * the driver supports the algorithms and mode requested by the + * session. + * + * If the driver does not support a session with the requested + * parameters, this function should fail with an error. + * + * If the driver does support a session with the requested parameters, + * this function should return a negative value indicating the + * priority of this driver. These negative values should be derived + * from one of the CRYPTODEV_PROBE_* constants in + * <opencrypto/cryptodev.h>. + * + * This function's return value is similar to that used by + * DEVICE_PROBE(9). However, a return value of zero is not supported + * and should not be used. + * + * @param dev the crypto driver device + * @param csp crypto session parameters + * + * @retval negative if the driver supports this session - the + * least negative value is used to select the + * driver for the session + * @retval EINVAL if the driver does not support the session + * @retval positive if some other error occurs + */ +METHOD int probesession { + device_t dev; + const struct crypto_session_params *csp; +}; + +/** + * @brief Initialize a new crypto session object + * + * Invoked by the crypto framework to initialize driver-specific data + * for a crypto session. The framework allocates and zeroes the + * driver's per-session memory object prior to invoking this method. + * The driver is able to access it's per-session memory object via * crypto_get_driver_session(). + * + * @param dev the crypto driver device + * @param crypto_session session being initialized + * @param csp crypto session parameters + * + * @retval 0 success + * @retval non-zero if some kind of error occurred */ METHOD int newsession { device_t dev; crypto_session_t crypto_session; - struct cryptoini *cri; + const struct crypto_session_params *csp; }; /** - * Optional crypto driver method to release any additional allocations. OCF - * owns session memory itself; it is zeroed before release. + * @brief Destroy a crypto session object + * + * The crypto framework invokes this method when tearing down a crypto + * session. After this callback returns, the frame will explicitly + * zero and free the drvier's per-session memory object. If the + * driver requires additional actions to destroy a session, it should + * perform those in this method. If the driver does not require + * additional actions it does not need to provide an implementation of + * this method. + * + * @param dev the crypto driver device + * @param crypto_session session being destroyed */ METHOD void freesession { device_t dev; crypto_session_t crypto_session; } DEFAULT null_freesession; +/** + * @brief Perform a symmetric crypto operation + * + * The crypto framework invokes this method for each symmetric crypto + * operation performed on a session. A reference to the containing + * session is stored as a member of 'struct cryptop'. This routine + * should not block, but queue the operation if necessary. + * + * This method may return ERESTART to indicate that any internal + * queues are full so the operation should be queued in the crypto + * framework and retried in the future. + * + * To report errors with a crypto operation, 'crp_etype' should be set + * and the operation completed by calling 'crypto_done'. This method + * should then return zero. + * + * @param dev the crypto driver device + * @param op crypto operation to perform + * @param flags set to CRYPTO_HINT_MORE if additional symmetric + * crypto operations are queued for this driver; + * otherwise set to zero. + * + * @retval 0 success + * @retval ERESTART internal queue is full + */ METHOD int process { device_t dev; struct cryptop *op; int flags; }; +/** + * @brief Perform an asymmetric crypto operation + * + * The crypto framework invokes this method for each asymmetric crypto + * operation. Each asymmetric crypto operation should be + * self-contained and is not assicated with any persistent session. + * This routine should not block, but queue the operation if + * necessary. + * + * This method may return ERESTART to indicate that any internal + * queues are full so the operation should be queued in the crypto + * framework and retried in the future. + * + * To report errors with a crypto operation, 'krp_status' should be set + * and the operation completed by calling 'crypto_kdone'. This method + * should then return zero. + * + * @param dev the crypto driver device + * @param op crypto operation to perform + * @param flags set to CRYPTO_HINT_MORE if additional asymmetric + * crypto operations are queued for this driver; + * otherwise set to zero. + * + * @retval 0 success + * @retval ERESTART internal queue is full + */ METHOD int kprocess { device_t dev; struct cryptkop *op; diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c index 2d064ffa15e8..e98f710a4e5b 100644 --- a/sys/opencrypto/cryptosoft.c +++ b/sys/opencrypto/cryptosoft.c @@ -55,36 +55,60 @@ __FBSDID("$FreeBSD$"); #include <sys/md5.h> #include <opencrypto/cryptodev.h> -#include <opencrypto/cryptosoft.h> #include <opencrypto/xform.h> #include <sys/kobj.h> #include <sys/bus.h> #include "cryptodev_if.h" -_Static_assert(AES_CCM_IV_LEN == AES_GCM_IV_LEN, - "AES_GCM_IV_LEN must currently be the same as AES_CCM_IV_LEN"); +struct swcr_auth { + void *sw_ictx; + void *sw_octx; + struct auth_hash *sw_axf; + uint16_t sw_mlen; + uint16_t sw_octx_len; +}; -static int32_t swcr_id; +struct swcr_encdec { + uint8_t *sw_kschedule; + struct enc_xform *sw_exf; +}; + +struct swcr_compdec { + struct comp_algo *sw_cxf; +}; -u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN]; -u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN]; +struct swcr_session { + struct mtx swcr_lock; + int (*swcr_process)(struct swcr_session *, struct cryptop *); + + struct swcr_auth swcr_auth; + struct swcr_encdec swcr_encdec; + struct swcr_compdec swcr_compdec; +}; + +static int32_t swcr_id; -static int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int); -static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int); -static int swcr_authenc(struct cryptop *crp); -static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int); static void swcr_freesession(device_t dev, crypto_session_t cses); +/* Used for CRYPTO_NULL_CBC. */ +static int +swcr_null(struct swcr_session *ses, struct cryptop *crp) +{ + + return (0); +} + /* * Apply a symmetric encryption/decryption algorithm. */ static int -swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, - int flags) +swcr_encdec(struct swcr_session *ses, struct cryptop *crp) { unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; + const struct crypto_session_params *csp; + struct swcr_encdec *sw; struct enc_xform *exf; int i, j, k, blks, ind, count, ivlen; struct uio *uio, uiolcl; @@ -92,51 +116,39 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, struct iovec *iov; int iovcnt, iovalloc; int error; + bool encrypting; error = 0; + sw = &ses->swcr_encdec; exf = sw->sw_exf; blks = exf->blocksize; ivlen = exf->ivsize; /* Check for non-padded data */ - if (crd->crd_len % blks) + if ((crp->crp_payload_length % blks) != 0) return EINVAL; - if (crd->crd_alg == CRYPTO_AES_ICM && - (crd->crd_flags & CRD_F_IV_EXPLICIT) == 0) + if (exf == &enc_xform_aes_icm && + (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); - /* Initialize the IV */ - if (crd->crd_flags & CRD_F_ENCRYPT) { - /* IV explicitly provided ? */ - if (crd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crd->crd_iv, iv, ivlen); - else - arc4rand(iv, ivlen, 0); - - /* Do we need to write the IV */ - if (!(crd->crd_flags & CRD_F_IV_PRESENT)) - crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv); - - } else { /* Decryption */ - /* IV explicitly provided ? */ - if (crd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crd->crd_iv, iv, ivlen); - else { - /* Get IV off buf */ - crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv); - } - } - - if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { - int error; + /* IV explicitly provided ? */ + if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + bcopy(crp->crp_iv, iv, ivlen); + else if (crp->crp_flags & CRYPTO_F_IV_GENERATE) { + arc4rand(iv, ivlen, 0); + crypto_copyback(crp, crp->crp_iv_start, ivlen, iv); + } else + crypto_copydata(crp, crp->crp_iv_start, ivlen, iv); + if (crp->crp_cipher_key != NULL) { if (sw->sw_kschedule) exf->zerokey(&(sw->sw_kschedule)); + csp = crypto_get_params(crp->crp_session); error = exf->setkey(&sw->sw_kschedule, - crd->crd_key, crd->crd_klen / 8); + crp->crp_cipher_key, csp->csp_cipher_klen); if (error) return (error); } @@ -145,20 +157,24 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, iovcnt = nitems(iovlcl); iovalloc = 0; uio = &uiolcl; - if ((flags & CRYPTO_F_IMBUF) != 0) { - error = crypto_mbuftoiov((struct mbuf *)buf, &iov, &iovcnt, + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + error = crypto_mbuftoiov(crp->crp_mbuf, &iov, &iovcnt, &iovalloc); if (error) return (error); uio->uio_iov = iov; uio->uio_iovcnt = iovcnt; - } else if ((flags & CRYPTO_F_IOV) != 0) - uio = (struct uio *)buf; - else { - iov[0].iov_base = buf; - iov[0].iov_len = crd->crd_skip + crd->crd_len; + break; + case CRYPTO_BUF_UIO: + uio = crp->crp_uio; + break; + case CRYPTO_BUF_CONTIG: + iov[0].iov_base = crp->crp_buf; + iov[0].iov_len = crp->crp_ilen; uio->uio_iov = iov; uio->uio_iovcnt = 1; + break; } ivp = iv; @@ -171,14 +187,15 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, exf->reinit(sw->sw_kschedule, iv); } - count = crd->crd_skip; + count = crp->crp_payload_start; ind = cuio_getptr(uio, count, &k); if (ind == -1) { error = EINVAL; goto out; } - i = crd->crd_len; + i = crp->crp_payload_length; + encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); while (i > 0) { /* @@ -191,14 +208,14 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, /* Actual encryption/decryption */ if (exf->reinit) { - if (crd->crd_flags & CRD_F_ENCRYPT) { + if (encrypting) { exf->encrypt(sw->sw_kschedule, blk); } else { exf->decrypt(sw->sw_kschedule, blk); } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { + } else if (encrypting) { /* XOR with previous block */ for (j = 0; j < blks; j++) blk[j] ^= ivp[j]; @@ -257,11 +274,10 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, idat = (uint8_t *)uio->uio_iov[ind].iov_base + k; if (exf->reinit) { - if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 && - exf->encrypt_multi == NULL) + if (encrypting && exf->encrypt_multi == NULL) exf->encrypt(sw->sw_kschedule, idat); - else if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) { + else if (encrypting) { nb = rounddown(rem, blks); exf->encrypt_multi(sw->sw_kschedule, idat, nb); @@ -273,7 +289,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, exf->decrypt_multi(sw->sw_kschedule, idat, nb); } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { + } else if (encrypting) { /* XOR with previous block/IV */ for (j = 0; j < blks; j++) idat[j] ^= ivp[j]; @@ -325,13 +341,10 @@ out: return (error); } -static int __result_use_check -swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key, - int klen) +static void +swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw, + const uint8_t *key, int klen) { - int k; - - klen /= 8; switch (axf->type) { case CRYPTO_MD5_HMAC: @@ -342,22 +355,8 @@ swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key, case CRYPTO_SHA2_512_HMAC: case CRYPTO_NULL_HMAC: case CRYPTO_RIPEMD160_HMAC: - for (k = 0; k < klen; k++) - key[k] ^= HMAC_IPAD_VAL; - - axf->Init(sw->sw_ictx); - axf->Update(sw->sw_ictx, key, klen); - axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen); - - for (k = 0; k < klen; k++) - key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); - - axf->Init(sw->sw_octx); - axf->Update(sw->sw_octx, key, klen); - axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen); - - for (k = 0; k < klen; k++) - key[k] ^= HMAC_OPAD_VAL; + hmac_init_ipad(axf, key, klen, sw->sw_ictx); + hmac_init_opad(axf, key, klen, sw->sw_octx); break; case CRYPTO_MD5_KPDK: case CRYPTO_SHA1_KPDK: @@ -374,7 +373,6 @@ swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key, */ u_char buf[SHA1_RESULTLEN]; - sw->sw_klen = klen; bcopy(key, sw->sw_octx, klen); axf->Init(sw->sw_ictx); axf->Update(sw->sw_ictx, key, klen); @@ -382,55 +380,53 @@ swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key, break; } case CRYPTO_POLY1305: - if (klen != POLY1305_KEY_LEN) { - CRYPTDEB("bad poly1305 key size %d", klen); - return EINVAL; - } - /* FALLTHROUGH */ case CRYPTO_BLAKE2B: case CRYPTO_BLAKE2S: axf->Setkey(sw->sw_ictx, key, klen); axf->Init(sw->sw_ictx); break; default: - printf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d " - "doesn't use keys.\n", __func__, axf->type); - return EINVAL; + panic("%s: algorithm %d doesn't use keys", __func__, axf->type); } - return 0; } /* - * Compute keyed-hash authenticator. + * Compute or verify hash. */ static int -swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, - int flags) +swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) { - unsigned char aalg[HASH_MAX_LEN]; + u_char aalg[HASH_MAX_LEN]; + u_char uaalg[HASH_MAX_LEN]; + const struct crypto_session_params *csp; + struct swcr_auth *sw; struct auth_hash *axf; union authctx ctx; int err; - if (sw->sw_ictx == 0) - return EINVAL; + sw = &ses->swcr_auth; axf = sw->sw_axf; - if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { - err = swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen); - if (err != 0) - return err; + if (crp->crp_auth_key != NULL) { + csp = crypto_get_params(crp->crp_session); + swcr_authprepare(axf, sw, crp->crp_auth_key, + csp->csp_auth_klen); } bcopy(sw->sw_ictx, &ctx, axf->ctxsize); - err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len, - (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx); + err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, + (int (*)(void *, void *, unsigned int))axf->Update, &ctx); if (err) return err; - switch (sw->sw_alg) { + err = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, + (int (*)(void *, void *, unsigned int))axf->Update, &ctx); + if (err) + return err; + + switch (axf->type) { case CRYPTO_SHA1: case CRYPTO_SHA2_224: case CRYPTO_SHA2_256: @@ -468,7 +464,7 @@ swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, * and let Final() do the proper, natural "algofill" * padding. */ - axf->Update(&ctx, sw->sw_octx, sw->sw_klen); + axf->Update(&ctx, sw->sw_octx, sw->sw_octx_len); axf->Final(aalg, &ctx); break; @@ -480,20 +476,22 @@ swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, break; } - /* Inject the authentication data */ - crypto_copyback(flags, buf, crd->crd_inject, - sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg); - return 0; + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); + if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) + return (EBADMSG); + } else { + /* Inject the authentication data */ + crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); + } + return (0); } CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ -/* - * Apply a combined encryption-authentication transformation - */ static int -swcr_authenc(struct cryptop *crp) +swcr_gmac(struct swcr_session *ses, struct cryptop *crp) { uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; u_char *blk = (u_char *)blkbuf; @@ -501,288 +499,403 @@ swcr_authenc(struct cryptop *crp) u_char uaalg[AALG_MAX_RESULT_LEN]; u_char iv[EALG_MAX_BLOCK_LEN]; union authctx ctx; - struct swcr_session *ses; - struct cryptodesc *crd, *crda = NULL, *crde = NULL; - struct swcr_data *sw, *swa, *swe = NULL; - struct auth_hash *axf = NULL; - struct enc_xform *exf = NULL; - caddr_t buf = (caddr_t)crp->crp_buf; + struct swcr_auth *swa; + struct auth_hash *axf; uint32_t *blkp; - int aadlen, blksz, i, ivlen, len, iskip, oskip, r; - int isccm = 0; + int blksz, i, ivlen, len; - ivlen = blksz = iskip = oskip = 0; + swa = &ses->swcr_auth; + axf = swa->sw_axf; - ses = crypto_get_driver_session(crp->crp_session); + bcopy(swa->sw_ictx, &ctx, axf->ctxsize); + blksz = axf->blocksize; - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - for (i = 0; i < nitems(ses->swcr_algorithms) && - ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++) - ; - if (i == nitems(ses->swcr_algorithms)) - return (EINVAL); + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) + return (EINVAL); - sw = &ses->swcr_algorithms[i]; - switch (sw->sw_alg) { - case CRYPTO_AES_CCM_16: - case CRYPTO_AES_NIST_GCM_16: - case CRYPTO_AES_NIST_GMAC: - swe = sw; - crde = crd; - exf = swe->sw_exf; - /* AES_CCM_IV_LEN and AES_GCM_IV_LEN are both 12 */ - ivlen = AES_CCM_IV_LEN; - break; - case CRYPTO_AES_CCM_CBC_MAC: - isccm = 1; - /* FALLTHROUGH */ - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - swa = sw; - crda = crd; - axf = swa->sw_axf; - if (swa->sw_ictx == 0) - return (EINVAL); - bcopy(swa->sw_ictx, &ctx, axf->ctxsize); - blksz = axf->blocksize; - break; - default: - return (EINVAL); - } + /* Initialize the IV */ + ivlen = AES_GCM_IV_LEN; + if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + bcopy(crp->crp_iv, iv, ivlen); + else + crypto_copydata(crp, crp->crp_iv_start, ivlen, iv); + + axf->Reinit(&ctx, iv, ivlen); + for (i = 0; i < crp->crp_payload_length; i += blksz) { + len = MIN(crp->crp_payload_length - i, blksz); + crypto_copydata(crp, crp->crp_payload_start + i, len, blk); + bzero(blk + len, blksz - len); + axf->Update(&ctx, blk, blksz); + } + + /* length block */ + bzero(blk, blksz); + blkp = (uint32_t *)blk + 1; + *blkp = htobe32(crp->crp_payload_length * 8); + axf->Update(&ctx, blk, blksz); + + /* Finalize MAC */ + axf->Final(aalg, &ctx); + + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, + uaalg); + if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0) + return (EBADMSG); + } else { + /* Inject the authentication data */ + crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg); } - if (crde == NULL || crda == NULL) + return (0); +} + +static int +swcr_gcm(struct swcr_session *ses, struct cryptop *crp) +{ + uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; + u_char *blk = (u_char *)blkbuf; + u_char aalg[AALG_MAX_RESULT_LEN]; + u_char uaalg[AALG_MAX_RESULT_LEN]; + u_char iv[EALG_MAX_BLOCK_LEN]; + union authctx ctx; + struct swcr_auth *swa; + struct swcr_encdec *swe; + struct auth_hash *axf; + struct enc_xform *exf; + uint32_t *blkp; + int blksz, i, ivlen, len, r; + + swa = &ses->swcr_auth; + axf = swa->sw_axf; + + bcopy(swa->sw_ictx, &ctx, axf->ctxsize); + blksz = axf->blocksize; + + swe = &ses->swcr_encdec; + exf = swe->sw_exf; + + if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) return (EINVAL); - /* - * We need to make sure that the auth algorithm matches the - * encr algorithm. Specifically, for AES-GCM must go with - * AES NIST GMAC, and AES-CCM must go with CBC-MAC. - */ - if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16) { - switch (crda->crd_alg) { - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - break; /* Good! */ - default: - return (EINVAL); /* Not good! */ + + /* Initialize the IV */ + ivlen = AES_GCM_IV_LEN; + bcopy(crp->crp_iv, iv, ivlen); + + /* Supply MAC with IV */ + axf->Reinit(&ctx, iv, ivlen); + + /* Supply MAC with AAD */ + for (i = 0; i < crp->crp_aad_length; i += blksz) { + len = MIN(crp->crp_aad_length - i, blksz); + crypto_copydata(crp, crp->crp_aad_start + i, len, blk); + bzero(blk + len, blksz - len); + axf->Update(&ctx, blk, blksz); + } + + exf->reinit(swe->sw_kschedule, iv); + + /* Do encryption with MAC */ + for (i = 0; i < crp->crp_payload_length; i += len) { + len = MIN(crp->crp_payload_length - i, blksz); + if (len < blksz) + bzero(blk, blksz); + crypto_copydata(crp, crp->crp_payload_start + i, len, blk); + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + exf->encrypt(swe->sw_kschedule, blk); + axf->Update(&ctx, blk, len); + crypto_copyback(crp, crp->crp_payload_start + i, len, + blk); + } else { + axf->Update(&ctx, blk, len); } - } else if (crde->crd_alg == CRYPTO_AES_CCM_16 && - crda->crd_alg != CRYPTO_AES_CCM_CBC_MAC) - return (EINVAL); + } - if ((crde->crd_alg == CRYPTO_AES_NIST_GCM_16 || - crde->crd_alg == CRYPTO_AES_CCM_16) && - (crde->crd_flags & CRD_F_IV_EXPLICIT) == 0) - return (EINVAL); + /* length block */ + bzero(blk, blksz); + blkp = (uint32_t *)blk + 1; + *blkp = htobe32(crp->crp_aad_length * 8); + blkp = (uint32_t *)blk + 3; + *blkp = htobe32(crp->crp_payload_length * 8); + axf->Update(&ctx, blk, blksz); + + /* Finalize MAC */ + axf->Final(aalg, &ctx); + + /* Validate tag */ + if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, + uaalg); + + r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen); + if (r != 0) + return (EBADMSG); + + /* tag matches, decrypt data */ + for (i = 0; i < crp->crp_payload_length; i += blksz) { + len = MIN(crp->crp_payload_length - i, blksz); + if (len < blksz) + bzero(blk, blksz); + crypto_copydata(crp, crp->crp_payload_start + i, len, + blk); + exf->decrypt(swe->sw_kschedule, blk); + crypto_copyback(crp, crp->crp_payload_start + i, len, + blk); + } + } else { + /* Inject the authentication data */ + crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, + aalg); + } + + return (0); +} - if (crde->crd_klen != crda->crd_klen) +static int +swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) +{ + uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; + u_char *blk = (u_char *)blkbuf; + u_char aalg[AALG_MAX_RESULT_LEN]; + u_char uaalg[AALG_MAX_RESULT_LEN]; + u_char iv[EALG_MAX_BLOCK_LEN]; + union authctx ctx; + struct swcr_auth *swa; + struct auth_hash *axf; + int blksz, i, ivlen, len; + + swa = &ses->swcr_auth; + axf = swa->sw_axf; + + bcopy(swa->sw_ictx, &ctx, axf->ctxsize); + blksz = axf->blocksize; + + if (crp->crp_flags & CRYPTO_F_IV_GENERATE) return (EINVAL); /* Initialize the IV */ - if (crde->crd_flags & CRD_F_ENCRYPT) { - /* IV explicitly provided ? */ - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crde->crd_iv, iv, ivlen); - else - arc4rand(iv, ivlen, 0); + ivlen = AES_CCM_IV_LEN; + if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) + bcopy(crp->crp_iv, iv, ivlen); + else + crypto_copydata(crp, crp->crp_iv_start, ivlen, iv); - /* Do we need to write the IV */ - if (!(crde->crd_flags & CRD_F_IV_PRESENT)) - crypto_copyback(crp->crp_flags, buf, crde->crd_inject, - ivlen, iv); + /* + * AES CCM-CBC-MAC needs to know the length of both the auth + * data and payload data before doing the auth computation. + */ + ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length; + ctx.aes_cbc_mac_ctx.cryptDataLength = 0; - } else { /* Decryption */ - /* IV explicitly provided ? */ - if (crde->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crde->crd_iv, iv, ivlen); - else { - /* Get IV off buf */ - crypto_copydata(crp->crp_flags, buf, crde->crd_inject, - ivlen, iv); - } + axf->Reinit(&ctx, iv, ivlen); + for (i = 0; i < crp->crp_payload_length; i += blksz) { + len = MIN(crp->crp_payload_length - i, blksz); + crypto_copydata(crp, crp->crp_payload_start + i, len, blk); + bzero(blk + len, blksz - len); + axf->Update(&ctx, blk, blksz); } - if (swa->sw_alg == CRYPTO_AES_CCM_CBC_MAC) { - /* - * AES CCM-CBC needs to know the length of - * both the auth data, and payload data, before - * doing the auth computation. - */ - ctx.aes_cbc_mac_ctx.authDataLength = crda->crd_len; - ctx.aes_cbc_mac_ctx.cryptDataLength = crde->crd_len; + /* Finalize MAC */ + axf->Final(aalg, &ctx); + + if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { + crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, + uaalg); + if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0) + return (EBADMSG); + } else { + /* Inject the authentication data */ + crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg); } + return (0); +} + +static int +swcr_ccm(struct swcr_session *ses, struct cryptop *crp) +{ + uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; + u_char *blk = (u_char *)blkbuf; + u_char aalg[AALG_MAX_RESULT_LEN]; + u_char uaalg[AALG_MAX_RESULT_LEN]; + u_char iv[EALG_MAX_BLOCK_LEN]; + union authctx ctx; + struct swcr_auth *swa; + struct swcr_encdec *swe; + struct auth_hash *axf; + struct enc_xform *exf; + int blksz, i, ivlen, len, r; + + swa = &ses->swcr_auth; + axf = swa->sw_axf; + + bcopy(swa->sw_ictx, &ctx, axf->ctxsize); + blksz = axf->blocksize; + + swe = &ses->swcr_encdec; + exf = swe->sw_exf; + + if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) + return (EINVAL); + + /* Initialize the IV */ + ivlen = AES_CCM_IV_LEN; + bcopy(crp->crp_iv, iv, ivlen); + + /* + * AES CCM-CBC-MAC needs to know the length of both the auth + * data and payload data before doing the auth computation. + */ + ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; + ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; + /* Supply MAC with IV */ - if (axf->Reinit) - axf->Reinit(&ctx, iv, ivlen); + axf->Reinit(&ctx, iv, ivlen); /* Supply MAC with AAD */ - aadlen = crda->crd_len; - - for (i = iskip; i < crda->crd_len; i += blksz) { - len = MIN(crda->crd_len - i, blksz - oskip); - crypto_copydata(crp->crp_flags, buf, crda->crd_skip + i, len, - blk + oskip); - bzero(blk + len + oskip, blksz - len - oskip); + for (i = 0; i < crp->crp_aad_length; i += blksz) { + len = MIN(crp->crp_aad_length - i, blksz); + crypto_copydata(crp, crp->crp_aad_start + i, len, blk); + bzero(blk + len, blksz - len); axf->Update(&ctx, blk, blksz); - oskip = 0; /* reset initial output offset */ } - if (exf->reinit) - exf->reinit(swe->sw_kschedule, iv); + exf->reinit(swe->sw_kschedule, iv); /* Do encryption/decryption with MAC */ - for (i = 0; i < crde->crd_len; i += len) { - if (exf->encrypt_multi != NULL) { - len = rounddown(crde->crd_len - i, blksz); - if (len == 0) - len = blksz; - else - len = MIN(len, sizeof(blkbuf)); - } else - len = blksz; - len = MIN(crde->crd_len - i, len); + for (i = 0; i < crp->crp_payload_length; i += len) { + len = MIN(crp->crp_payload_length - i, blksz); if (len < blksz) bzero(blk, blksz); - crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len, - blk); - /* - * One of the problems with CCM+CBC is that the authentication - * is done on the unecncrypted data. As a result, we have - * to do the authentication update at different times, - * depending on whether it's CCM or not. - */ - if (crde->crd_flags & CRD_F_ENCRYPT) { - if (isccm) - axf->Update(&ctx, blk, len); - if (exf->encrypt_multi != NULL) - exf->encrypt_multi(swe->sw_kschedule, blk, - len); - else - exf->encrypt(swe->sw_kschedule, blk); - if (!isccm) - axf->Update(&ctx, blk, len); - crypto_copyback(crp->crp_flags, buf, - crde->crd_skip + i, len, blk); + crypto_copydata(crp, crp->crp_payload_start + i, len, blk); + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + axf->Update(&ctx, blk, len); + exf->encrypt(swe->sw_kschedule, blk); + crypto_copyback(crp, crp->crp_payload_start + i, len, + blk); } else { - if (isccm) { - KASSERT(exf->encrypt_multi == NULL, - ("assume CCM is single-block only")); - exf->decrypt(swe->sw_kschedule, blk); - } + /* + * One of the problems with CCM+CBC is that + * the authentication is done on the + * unecncrypted data. As a result, we have to + * decrypt the data twice: once to generate + * the tag and a second time after the tag is + * verified. + */ + exf->decrypt(swe->sw_kschedule, blk); axf->Update(&ctx, blk, len); } } - /* Do any required special finalization */ - switch (crda->crd_alg) { - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - /* length block */ - bzero(blk, blksz); - blkp = (uint32_t *)blk + 1; - *blkp = htobe32(aadlen * 8); - blkp = (uint32_t *)blk + 3; - *blkp = htobe32(crde->crd_len * 8); - axf->Update(&ctx, blk, blksz); - break; - } - /* Finalize MAC */ axf->Final(aalg, &ctx); /* Validate tag */ - if (!(crde->crd_flags & CRD_F_ENCRYPT)) { - crypto_copydata(crp->crp_flags, buf, crda->crd_inject, - axf->hashsize, uaalg); + if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, + uaalg); - r = timingsafe_bcmp(aalg, uaalg, axf->hashsize); - if (r == 0) { - /* tag matches, decrypt data */ - if (isccm) { - KASSERT(exf->reinit != NULL, - ("AES-CCM reinit function must be set")); - exf->reinit(swe->sw_kschedule, iv); - } - for (i = 0; i < crde->crd_len; i += blksz) { - len = MIN(crde->crd_len - i, blksz); - if (len < blksz) - bzero(blk, blksz); - crypto_copydata(crp->crp_flags, buf, - crde->crd_skip + i, len, blk); - exf->decrypt(swe->sw_kschedule, blk); - crypto_copyback(crp->crp_flags, buf, - crde->crd_skip + i, len, blk); - } - } else + r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen); + if (r != 0) return (EBADMSG); + + /* tag matches, decrypt data */ + exf->reinit(swe->sw_kschedule, iv); + for (i = 0; i < crp->crp_payload_length; i += blksz) { + len = MIN(crp->crp_payload_length - i, blksz); + if (len < blksz) + bzero(blk, blksz); + crypto_copydata(crp, crp->crp_payload_start + i, len, + blk); + exf->decrypt(swe->sw_kschedule, blk); + crypto_copyback(crp, crp->crp_payload_start + i, len, + blk); + } } else { /* Inject the authentication data */ - crypto_copyback(crp->crp_flags, buf, crda->crd_inject, - axf->hashsize, aalg); + crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, + aalg); } return (0); } /* + * Apply a cipher and a digest to perform EtA. + */ +static int +swcr_eta(struct swcr_session *ses, struct cryptop *crp) +{ + int error; + + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + error = swcr_encdec(ses, crp); + if (error == 0) + error = swcr_authcompute(ses, crp); + } else { + error = swcr_authcompute(ses, crp); + if (error == 0) + error = swcr_encdec(ses, crp); + } + return (error); +} + +/* * Apply a compression/decompression algorithm */ static int -swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw, - caddr_t buf, int flags) +swcr_compdec(struct swcr_session *ses, struct cryptop *crp) { u_int8_t *data, *out; struct comp_algo *cxf; int adj; u_int32_t result; - cxf = sw->sw_cxf; + cxf = ses->swcr_compdec.sw_cxf; /* We must handle the whole buffer of data in one time * then if there is not all the data in the mbuf, we must * copy in a buffer. */ - data = malloc(crd->crd_len, M_CRYPTO_DATA, M_NOWAIT); + data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); if (data == NULL) return (EINVAL); - crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data); + crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, + data); - if (crd->crd_flags & CRD_F_COMP) - result = cxf->compress(data, crd->crd_len, &out); + if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) + result = cxf->compress(data, crp->crp_payload_length, &out); else - result = cxf->decompress(data, crd->crd_len, &out); + result = cxf->decompress(data, crp->crp_payload_length, &out); free(data, M_CRYPTO_DATA); if (result == 0) - return EINVAL; + return (EINVAL); + crp->crp_olen = result; - /* Copy back the (de)compressed data. m_copyback is - * extending the mbuf as necessary. - */ - sw->sw_size = result; /* Check the compressed size when doing compression */ - if (crd->crd_flags & CRD_F_COMP) { - if (result >= crd->crd_len) { + if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { + if (result >= crp->crp_payload_length) { /* Compression was useless, we lost time */ free(out, M_CRYPTO_DATA); - return 0; + return (0); } } - crypto_copyback(flags, buf, crd->crd_skip, result, out); - if (result < crd->crd_len) { - adj = result - crd->crd_len; - if (flags & CRYPTO_F_IMBUF) { - adj = result - crd->crd_len; - m_adj((struct mbuf *)buf, adj); - } else if (flags & CRYPTO_F_IOV) { - struct uio *uio = (struct uio *)buf; + /* Copy back the (de)compressed data. m_copyback is + * extending the mbuf as necessary. + */ + crypto_copyback(crp, crp->crp_payload_start, result, out); + if (result < crp->crp_payload_length) { + switch (crp->crp_buf_type) { + case CRYPTO_BUF_MBUF: + adj = result - crp->crp_payload_length; + m_adj(crp->crp_mbuf, adj); + break; + case CRYPTO_BUF_UIO: { + struct uio *uio = crp->crp_uio; int ind; - adj = crd->crd_len - result; + adj = crp->crp_payload_length - result; ind = uio->uio_iovcnt - 1; while (adj > 0 && ind >= 0) { @@ -796,391 +909,522 @@ swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw, ind--; uio->uio_iovcnt--; } + } + break; } } free(out, M_CRYPTO_DATA); return 0; } -/* - * Generate a new software session. - */ static int -swcr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) +swcr_setup_encdec(struct swcr_session *ses, + const struct crypto_session_params *csp) { - struct swcr_session *ses; - struct swcr_data *swd; - struct auth_hash *axf; + struct swcr_encdec *swe; struct enc_xform *txf; - struct comp_algo *cxf; - size_t i; - int len; int error; - if (cses == NULL || cri == NULL) - return EINVAL; + swe = &ses->swcr_encdec; + txf = crypto_cipher(csp); + MPASS(txf->ivsize == csp->csp_ivlen); + if (csp->csp_cipher_key != NULL) { + error = txf->setkey(&swe->sw_kschedule, + csp->csp_cipher_key, csp->csp_cipher_klen); + if (error) + return (error); + } + swe->sw_exf = txf; + return (0); +} - ses = crypto_get_driver_session(cses); - mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); +static int +swcr_setup_auth(struct swcr_session *ses, + const struct crypto_session_params *csp) +{ + struct swcr_auth *swa; + struct auth_hash *axf; - for (i = 0; cri != NULL && i < nitems(ses->swcr_algorithms); i++) { - swd = &ses->swcr_algorithms[i]; + swa = &ses->swcr_auth; - switch (cri->cri_alg) { - case CRYPTO_DES_CBC: - txf = &enc_xform_des; - goto enccommon; - case CRYPTO_3DES_CBC: - txf = &enc_xform_3des; - goto enccommon; - case CRYPTO_BLF_CBC: - txf = &enc_xform_blf; - goto enccommon; - case CRYPTO_CAST_CBC: - txf = &enc_xform_cast5; - goto enccommon; - case CRYPTO_SKIPJACK_CBC: - txf = &enc_xform_skipjack; - goto enccommon; - case CRYPTO_RIJNDAEL128_CBC: - txf = &enc_xform_rijndael128; - goto enccommon; - case CRYPTO_AES_XTS: - txf = &enc_xform_aes_xts; - goto enccommon; - case CRYPTO_AES_ICM: - txf = &enc_xform_aes_icm; - goto enccommon; - case CRYPTO_AES_NIST_GCM_16: - txf = &enc_xform_aes_nist_gcm; - goto enccommon; - case CRYPTO_AES_CCM_16: - txf = &enc_xform_ccm; - goto enccommon; - case CRYPTO_AES_NIST_GMAC: - txf = &enc_xform_aes_nist_gmac; - swd->sw_exf = txf; - break; - case CRYPTO_CAMELLIA_CBC: - txf = &enc_xform_camellia; - goto enccommon; - case CRYPTO_NULL_CBC: - txf = &enc_xform_null; - goto enccommon; - case CRYPTO_CHACHA20: - txf = &enc_xform_chacha20; - goto enccommon; - enccommon: - if (cri->cri_key != NULL) { - error = txf->setkey(&swd->sw_kschedule, - cri->cri_key, cri->cri_klen / 8); - if (error) { - swcr_freesession(dev, cses); - return error; - } - } - swd->sw_exf = txf; - break; - - case CRYPTO_MD5_HMAC: - axf = &auth_hash_hmac_md5; - goto authcommon; - case CRYPTO_SHA1_HMAC: - axf = &auth_hash_hmac_sha1; - goto authcommon; - case CRYPTO_SHA2_224_HMAC: - axf = &auth_hash_hmac_sha2_224; - goto authcommon; - case CRYPTO_SHA2_256_HMAC: - axf = &auth_hash_hmac_sha2_256; - goto authcommon; - case CRYPTO_SHA2_384_HMAC: - axf = &auth_hash_hmac_sha2_384; - goto authcommon; - case CRYPTO_SHA2_512_HMAC: - axf = &auth_hash_hmac_sha2_512; - goto authcommon; - case CRYPTO_NULL_HMAC: - axf = &auth_hash_null; - goto authcommon; - case CRYPTO_RIPEMD160_HMAC: - axf = &auth_hash_hmac_ripemd_160; - authcommon: - swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, - M_NOWAIT); - if (swd->sw_ictx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } + axf = crypto_auth_hash(csp); + swa->sw_axf = axf; + if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) + return (EINVAL); + if (csp->csp_auth_mlen == 0) + swa->sw_mlen = axf->hashsize; + else + swa->sw_mlen = csp->csp_auth_mlen; + swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); + if (swa->sw_ictx == NULL) + return (ENOBUFS); - swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, - M_NOWAIT); - if (swd->sw_octx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } + switch (csp->csp_auth_alg) { + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + case CRYPTO_NULL_HMAC: + case CRYPTO_RIPEMD160_HMAC: + swa->sw_octx_len = axf->ctxsize; + swa->sw_octx = malloc(swa->sw_octx_len, M_CRYPTO_DATA, + M_NOWAIT); + if (swa->sw_octx == NULL) + return (ENOBUFS); - if (cri->cri_key != NULL) { - error = swcr_authprepare(axf, swd, - cri->cri_key, cri->cri_klen); - if (error != 0) { - swcr_freesession(dev, cses); - return error; - } - } + if (csp->csp_auth_key != NULL) { + swcr_authprepare(axf, swa, csp->csp_auth_key, + csp->csp_auth_klen); + } - swd->sw_mlen = cri->cri_mlen; - swd->sw_axf = axf; - break; - - case CRYPTO_MD5_KPDK: - axf = &auth_hash_key_md5; - goto auth2common; - - case CRYPTO_SHA1_KPDK: - axf = &auth_hash_key_sha1; - auth2common: - swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, - M_NOWAIT); - if (swd->sw_ictx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } - - swd->sw_octx = malloc(cri->cri_klen / 8, - M_CRYPTO_DATA, M_NOWAIT); - if (swd->sw_octx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } + if (csp->csp_mode == CSP_MODE_DIGEST) + ses->swcr_process = swcr_authcompute; + break; + case CRYPTO_MD5_KPDK: + case CRYPTO_SHA1_KPDK: + swa->sw_octx_len = csp->csp_auth_klen; + swa->sw_octx = malloc(swa->sw_octx_len, M_CRYPTO_DATA, + M_NOWAIT); + if (swa->sw_octx == NULL) + return (ENOBUFS); - /* Store the key so we can "append" it to the payload */ - if (cri->cri_key != NULL) { - error = swcr_authprepare(axf, swd, - cri->cri_key, cri->cri_klen); - if (error != 0) { - swcr_freesession(dev, cses); - return error; - } - } + /* Store the key so we can "append" it to the payload */ + if (csp->csp_auth_key != NULL) { + swcr_authprepare(axf, swa, csp->csp_auth_key, + csp->csp_auth_klen); + } - swd->sw_mlen = cri->cri_mlen; - swd->sw_axf = axf; - break; + if (csp->csp_mode == CSP_MODE_DIGEST) + ses->swcr_process = swcr_authcompute; + break; #ifdef notdef - case CRYPTO_MD5: - axf = &auth_hash_md5; - goto auth3common; + case CRYPTO_MD5: #endif + case CRYPTO_SHA1: + case CRYPTO_SHA2_224: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_512: + axf->Init(swa->sw_ictx); + if (csp->csp_mode == CSP_MODE_DIGEST) + ses->swcr_process = swcr_authcompute; + break; + case CRYPTO_AES_NIST_GMAC: + axf->Init(swa->sw_ictx); + axf->Setkey(swa->sw_ictx, csp->csp_auth_key, + csp->csp_auth_klen); + if (csp->csp_mode == CSP_MODE_DIGEST) + ses->swcr_process = swcr_gmac; + break; + case CRYPTO_POLY1305: + case CRYPTO_BLAKE2B: + case CRYPTO_BLAKE2S: + /* + * Blake2b and Blake2s support an optional key but do + * not require one. + */ + if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) + axf->Setkey(swa->sw_ictx, csp->csp_auth_key, + csp->csp_auth_klen); + axf->Init(swa->sw_ictx); + if (csp->csp_mode == CSP_MODE_DIGEST) + ses->swcr_process = swcr_authcompute; + break; + case CRYPTO_AES_CCM_CBC_MAC: + axf->Init(swa->sw_ictx); + axf->Setkey(swa->sw_ictx, csp->csp_auth_key, + csp->csp_auth_klen); + if (csp->csp_mode == CSP_MODE_DIGEST) + ses->swcr_process = swcr_ccm_cbc_mac; + break; + } - case CRYPTO_SHA1: - axf = &auth_hash_sha1; - goto auth3common; - case CRYPTO_SHA2_224: - axf = &auth_hash_sha2_224; - goto auth3common; - case CRYPTO_SHA2_256: - axf = &auth_hash_sha2_256; - goto auth3common; - case CRYPTO_SHA2_384: - axf = &auth_hash_sha2_384; - goto auth3common; - case CRYPTO_SHA2_512: - axf = &auth_hash_sha2_512; + return (0); +} - auth3common: - swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, - M_NOWAIT); - if (swd->sw_ictx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } +static int +swcr_setup_gcm(struct swcr_session *ses, + const struct crypto_session_params *csp) +{ + struct swcr_encdec *swe; + struct swcr_auth *swa; + struct enc_xform *txf; + struct auth_hash *axf; + int error; - axf->Init(swd->sw_ictx); - swd->sw_mlen = cri->cri_mlen; - swd->sw_axf = axf; - break; + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); - case CRYPTO_AES_CCM_CBC_MAC: - switch (cri->cri_klen) { - case 128: - axf = &auth_hash_ccm_cbc_mac_128; - break; - case 192: - axf = &auth_hash_ccm_cbc_mac_192; - break; - case 256: - axf = &auth_hash_ccm_cbc_mac_256; - break; - default: - swcr_freesession(dev, cses); - return EINVAL; - } - goto auth4common; - case CRYPTO_AES_128_NIST_GMAC: - axf = &auth_hash_nist_gmac_aes_128; - goto auth4common; + /* First, setup the auth side. */ + swa = &ses->swcr_auth; + switch (csp->csp_cipher_klen * 8) { + case 128: + axf = &auth_hash_nist_gmac_aes_128; + break; + case 192: + axf = &auth_hash_nist_gmac_aes_192; + break; + case 256: + axf = &auth_hash_nist_gmac_aes_256; + break; + default: + return (EINVAL); + } + swa->sw_axf = axf; + if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) + return (EINVAL); + if (csp->csp_auth_mlen == 0) + swa->sw_mlen = axf->hashsize; + else + swa->sw_mlen = csp->csp_auth_mlen; + swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); + if (swa->sw_ictx == NULL) + return (ENOBUFS); + axf->Init(swa->sw_ictx); + if (csp->csp_cipher_key != NULL) + axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, + csp->csp_cipher_klen); - case CRYPTO_AES_192_NIST_GMAC: - axf = &auth_hash_nist_gmac_aes_192; - goto auth4common; + /* Second, setup the cipher side. */ + swe = &ses->swcr_encdec; + txf = &enc_xform_aes_nist_gcm; + if (csp->csp_cipher_key != NULL) { + error = txf->setkey(&swe->sw_kschedule, + csp->csp_cipher_key, csp->csp_cipher_klen); + if (error) + return (error); + } + swe->sw_exf = txf; - case CRYPTO_AES_256_NIST_GMAC: - axf = &auth_hash_nist_gmac_aes_256; - auth4common: - len = cri->cri_klen / 8; - if (len != 16 && len != 24 && len != 32) { - swcr_freesession(dev, cses); - return EINVAL; - } + return (0); +} - swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, - M_NOWAIT); - if (swd->sw_ictx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } - axf->Init(swd->sw_ictx); - axf->Setkey(swd->sw_ictx, cri->cri_key, len); - swd->sw_axf = axf; - break; +static int +swcr_setup_ccm(struct swcr_session *ses, + const struct crypto_session_params *csp) +{ + struct swcr_encdec *swe; + struct swcr_auth *swa; + struct enc_xform *txf; + struct auth_hash *axf; + int error; - case CRYPTO_BLAKE2B: - axf = &auth_hash_blake2b; - goto auth5common; - case CRYPTO_BLAKE2S: - axf = &auth_hash_blake2s; - goto auth5common; - case CRYPTO_POLY1305: - axf = &auth_hash_poly1305; - auth5common: - swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, - M_NOWAIT); - if (swd->sw_ictx == NULL) { - swcr_freesession(dev, cses); - return ENOBUFS; - } - axf->Setkey(swd->sw_ictx, cri->cri_key, - cri->cri_klen / 8); - axf->Init(swd->sw_ictx); - swd->sw_axf = axf; + if (csp->csp_ivlen != AES_CCM_IV_LEN) + return (EINVAL); + + /* First, setup the auth side. */ + swa = &ses->swcr_auth; + switch (csp->csp_cipher_klen * 8) { + case 128: + axf = &auth_hash_ccm_cbc_mac_128; + break; + case 192: + axf = &auth_hash_ccm_cbc_mac_192; + break; + case 256: + axf = &auth_hash_ccm_cbc_mac_256; + break; + default: + return (EINVAL); + } + swa->sw_axf = axf; + if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) + return (EINVAL); + if (csp->csp_auth_mlen == 0) + swa->sw_mlen = axf->hashsize; + else + swa->sw_mlen = csp->csp_auth_mlen; + swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); + if (swa->sw_ictx == NULL) + return (ENOBUFS); + axf->Init(swa->sw_ictx); + if (csp->csp_cipher_key != NULL) + axf->Setkey(swa->sw_ictx, csp->csp_cipher_key, + csp->csp_cipher_klen); + + /* Second, setup the cipher side. */ + swe = &ses->swcr_encdec; + txf = &enc_xform_ccm; + if (csp->csp_cipher_key != NULL) { + error = txf->setkey(&swe->sw_kschedule, + csp->csp_cipher_key, csp->csp_cipher_klen); + if (error) + return (error); + } + swe->sw_exf = txf; + + return (0); +} + +static bool +swcr_auth_supported(const struct crypto_session_params *csp) +{ + struct auth_hash *axf; + + axf = crypto_auth_hash(csp); + if (axf == NULL) + return (false); + switch (csp->csp_auth_alg) { + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + case CRYPTO_NULL_HMAC: + case CRYPTO_RIPEMD160_HMAC: + case CRYPTO_MD5_KPDK: + case CRYPTO_SHA1_KPDK: + break; + case CRYPTO_AES_NIST_GMAC: + switch (csp->csp_auth_klen * 8) { + case 128: + case 192: + case 256: + break; + default: + return (false); + } + if (csp->csp_auth_key == NULL) + return (false); + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (false); + break; + case CRYPTO_POLY1305: + if (csp->csp_auth_klen != POLY1305_KEY_LEN) + return (false); + break; + case CRYPTO_AES_CCM_CBC_MAC: + switch (csp->csp_auth_klen * 8) { + case 128: + case 192: + case 256: break; + default: + return (false); + } + if (csp->csp_auth_key == NULL) + return (false); + if (csp->csp_ivlen != AES_CCM_IV_LEN) + return (false); + break; + } + return (true); +} + +static bool +swcr_cipher_supported(const struct crypto_session_params *csp) +{ + struct enc_xform *txf; + + txf = crypto_cipher(csp); + if (txf == NULL) + return (false); + if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && + txf->ivsize != csp->csp_ivlen) + return (false); + return (true); +} + +static int +swcr_probesession(device_t dev, const struct crypto_session_params *csp) +{ + if (csp->csp_flags != 0) + return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_COMPRESS: + switch (csp->csp_cipher_alg) { case CRYPTO_DEFLATE_COMP: - cxf = &comp_algo_deflate; - swd->sw_cxf = cxf; break; default: - swcr_freesession(dev, cses); - return EINVAL; + return (EINVAL); + } + break; + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: + return (EINVAL); + default: + if (!swcr_cipher_supported(csp)) + return (EINVAL); + break; + } + break; + case CSP_MODE_DIGEST: + if (!swcr_auth_supported(csp)) + return (EINVAL); + break; + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: + break; + default: + return (EINVAL); + } + break; + case CSP_MODE_ETA: + /* AEAD algorithms cannot be used for EtA. */ + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: + return (EINVAL); + } + switch (csp->csp_auth_alg) { + case CRYPTO_AES_NIST_GMAC: + case CRYPTO_AES_CCM_CBC_MAC: + return (EINVAL); } - - swd->sw_alg = cri->cri_alg; - cri = cri->cri_next; - ses->swcr_nalgs++; - } - if (cri != NULL) { - CRYPTDEB("Bogus session request for three or more algorithms"); - return EINVAL; + if (!swcr_cipher_supported(csp) || + !swcr_auth_supported(csp)) + return (EINVAL); + break; + default: + return (EINVAL); } - return 0; + + return (CRYPTODEV_PROBE_SOFTWARE); } -static void -swcr_freesession(device_t dev, crypto_session_t cses) +/* + * Generate a new software session. + */ +static int +swcr_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) { struct swcr_session *ses; - struct swcr_data *swd; - struct enc_xform *txf; - struct auth_hash *axf; - size_t i; + struct swcr_encdec *swe; + struct swcr_auth *swa; + struct comp_algo *cxf; + int error; ses = crypto_get_driver_session(cses); + mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); - mtx_destroy(&ses->swcr_lock); - for (i = 0; i < nitems(ses->swcr_algorithms); i++) { - swd = &ses->swcr_algorithms[i]; - - switch (swd->sw_alg) { - case CRYPTO_DES_CBC: - case CRYPTO_3DES_CBC: - case CRYPTO_BLF_CBC: - case CRYPTO_CAST_CBC: - case CRYPTO_SKIPJACK_CBC: - case CRYPTO_RIJNDAEL128_CBC: - case CRYPTO_AES_XTS: - case CRYPTO_AES_ICM: - case CRYPTO_AES_NIST_GCM_16: - case CRYPTO_AES_NIST_GMAC: - case CRYPTO_CAMELLIA_CBC: + error = 0; + swe = &ses->swcr_encdec; + swa = &ses->swcr_auth; + switch (csp->csp_mode) { + case CSP_MODE_COMPRESS: + switch (csp->csp_cipher_alg) { + case CRYPTO_DEFLATE_COMP: + cxf = &comp_algo_deflate; + break; +#ifdef INVARIANTS + default: + panic("bad compression algo"); +#endif + } + ses->swcr_compdec.sw_cxf = cxf; + ses->swcr_process = swcr_compdec; + break; + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { case CRYPTO_NULL_CBC: - case CRYPTO_CHACHA20: + ses->swcr_process = swcr_null; + break; +#ifdef INVARIANTS + case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_CCM_16: - txf = swd->sw_exf; - - if (swd->sw_kschedule) - txf->zerokey(&(swd->sw_kschedule)); + panic("bad cipher algo"); +#endif + default: + error = swcr_setup_encdec(ses, csp); + if (error == 0) + ses->swcr_process = swcr_encdec; + } + break; + case CSP_MODE_DIGEST: + error = swcr_setup_auth(ses, csp); + break; + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + error = swcr_setup_gcm(ses, csp); + if (error == 0) + ses->swcr_process = swcr_gcm; break; - - case CRYPTO_MD5_HMAC: - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_224_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - case CRYPTO_RIPEMD160_HMAC: - case CRYPTO_NULL_HMAC: + case CRYPTO_AES_CCM_16: + error = swcr_setup_ccm(ses, csp); + if (error == 0) + ses->swcr_process = swcr_ccm; + break; +#ifdef INVARIANTS + default: + panic("bad aead algo"); +#endif + } + break; + case CSP_MODE_ETA: +#ifdef INVARIANTS + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_CCM_16: + panic("bad eta cipher algo"); + } + switch (csp->csp_auth_alg) { + case CRYPTO_AES_NIST_GMAC: case CRYPTO_AES_CCM_CBC_MAC: - axf = swd->sw_axf; + panic("bad eta auth algo"); + } +#endif - if (swd->sw_ictx) { - bzero(swd->sw_ictx, axf->ctxsize); - free(swd->sw_ictx, M_CRYPTO_DATA); - } - if (swd->sw_octx) { - bzero(swd->sw_octx, axf->ctxsize); - free(swd->sw_octx, M_CRYPTO_DATA); - } + error = swcr_setup_auth(ses, csp); + if (error) break; + if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { + /* Effectively degrade to digest mode. */ + ses->swcr_process = swcr_authcompute; + break; + } - case CRYPTO_MD5_KPDK: - case CRYPTO_SHA1_KPDK: - axf = swd->sw_axf; + error = swcr_setup_encdec(ses, csp); + if (error == 0) + ses->swcr_process = swcr_eta; + break; + default: + error = EINVAL; + } - if (swd->sw_ictx) { - bzero(swd->sw_ictx, axf->ctxsize); - free(swd->sw_ictx, M_CRYPTO_DATA); - } - if (swd->sw_octx) { - bzero(swd->sw_octx, swd->sw_klen); - free(swd->sw_octx, M_CRYPTO_DATA); - } - break; + if (error) + swcr_freesession(dev, cses); + return (error); +} + +static void +swcr_freesession(device_t dev, crypto_session_t cses) +{ + struct swcr_session *ses; + struct swcr_auth *swa; + struct enc_xform *txf; + struct auth_hash *axf; - case CRYPTO_BLAKE2B: - case CRYPTO_BLAKE2S: - case CRYPTO_MD5: - case CRYPTO_POLY1305: - case CRYPTO_SHA1: - case CRYPTO_SHA2_224: - case CRYPTO_SHA2_256: - case CRYPTO_SHA2_384: - case CRYPTO_SHA2_512: - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - axf = swd->sw_axf; + ses = crypto_get_driver_session(cses); - if (swd->sw_ictx) { - explicit_bzero(swd->sw_ictx, axf->ctxsize); - free(swd->sw_ictx, M_CRYPTO_DATA); - } - break; + mtx_destroy(&ses->swcr_lock); - case CRYPTO_DEFLATE_COMP: - /* Nothing to do */ - break; + txf = ses->swcr_encdec.sw_exf; + if (txf != NULL) { + if (ses->swcr_encdec.sw_kschedule != NULL) + txf->zerokey(&(ses->swcr_encdec.sw_kschedule)); + } + + axf = ses->swcr_auth.sw_axf; + if (axf != NULL) { + swa = &ses->swcr_auth; + if (swa->sw_ictx != NULL) { + explicit_bzero(swa->sw_ictx, axf->ctxsize); + free(swa->sw_ictx, M_CRYPTO_DATA); + } + if (swa->sw_octx != NULL) { + explicit_bzero(swa->sw_octx, swa->sw_octx_len); + free(swa->sw_octx, M_CRYPTO_DATA); } } } @@ -1191,117 +1435,16 @@ swcr_freesession(device_t dev, crypto_session_t cses) static int swcr_process(device_t dev, struct cryptop *crp, int hint) { - struct swcr_session *ses = NULL; - struct cryptodesc *crd; - struct swcr_data *sw; - size_t i; - - /* Sanity check */ - if (crp == NULL) - return EINVAL; - - if (crp->crp_desc == NULL || crp->crp_buf == NULL) { - crp->crp_etype = EINVAL; - goto done; - } + struct swcr_session *ses; ses = crypto_get_driver_session(crp->crp_session); mtx_lock(&ses->swcr_lock); - /* Go through crypto descriptors, processing as we go */ - for (crd = crp->crp_desc; crd; crd = crd->crd_next) { - /* - * Find the crypto context. - * - * XXX Note that the logic here prevents us from having - * XXX the same algorithm multiple times in a session - * XXX (or rather, we can but it won't give us the right - * XXX results). To do that, we'd need some way of differentiating - * XXX between the various instances of an algorithm (so we can - * XXX locate the correct crypto context). - */ - for (i = 0; i < nitems(ses->swcr_algorithms) && - ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++) - ; - - /* No such context ? */ - if (i == nitems(ses->swcr_algorithms)) { - crp->crp_etype = EINVAL; - goto done; - } - sw = &ses->swcr_algorithms[i]; - switch (sw->sw_alg) { - case CRYPTO_DES_CBC: - case CRYPTO_3DES_CBC: - case CRYPTO_BLF_CBC: - case CRYPTO_CAST_CBC: - case CRYPTO_SKIPJACK_CBC: - case CRYPTO_RIJNDAEL128_CBC: - case CRYPTO_AES_XTS: - case CRYPTO_AES_ICM: - case CRYPTO_CAMELLIA_CBC: - case CRYPTO_CHACHA20: - if ((crp->crp_etype = swcr_encdec(crd, sw, - crp->crp_buf, crp->crp_flags)) != 0) - goto done; - break; - case CRYPTO_NULL_CBC: - crp->crp_etype = 0; - break; - case CRYPTO_MD5_HMAC: - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_224_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - case CRYPTO_RIPEMD160_HMAC: - case CRYPTO_NULL_HMAC: - case CRYPTO_MD5_KPDK: - case CRYPTO_SHA1_KPDK: - case CRYPTO_MD5: - case CRYPTO_SHA1: - case CRYPTO_SHA2_224: - case CRYPTO_SHA2_256: - case CRYPTO_SHA2_384: - case CRYPTO_SHA2_512: - case CRYPTO_BLAKE2B: - case CRYPTO_BLAKE2S: - case CRYPTO_POLY1305: - if ((crp->crp_etype = swcr_authcompute(crd, sw, - crp->crp_buf, crp->crp_flags)) != 0) - goto done; - break; - - case CRYPTO_AES_NIST_GCM_16: - case CRYPTO_AES_NIST_GMAC: - case CRYPTO_AES_128_NIST_GMAC: - case CRYPTO_AES_192_NIST_GMAC: - case CRYPTO_AES_256_NIST_GMAC: - case CRYPTO_AES_CCM_16: - case CRYPTO_AES_CCM_CBC_MAC: - crp->crp_etype = swcr_authenc(crp); - goto done; - - case CRYPTO_DEFLATE_COMP: - if ((crp->crp_etype = swcr_compdec(crd, sw, - crp->crp_buf, crp->crp_flags)) != 0) - goto done; - else - crp->crp_olen = (int)sw->sw_size; - break; - - default: - /* Unknown/unsupported algorithm */ - crp->crp_etype = EINVAL; - goto done; - } - } + crp->crp_etype = ses->swcr_process(ses, crp); -done: - if (ses) - mtx_unlock(&ses->swcr_lock); + mtx_unlock(&ses->swcr_lock); crypto_done(crp); - return 0; + return (0); } static void @@ -1323,58 +1466,15 @@ swcr_probe(device_t dev) static int swcr_attach(device_t dev) { - memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN); - memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN); swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); if (swcr_id < 0) { device_printf(dev, "cannot initialize!"); - return ENOMEM; + return (ENXIO); } -#define REGISTER(alg) \ - crypto_register(swcr_id, alg, 0,0) - REGISTER(CRYPTO_DES_CBC); - REGISTER(CRYPTO_3DES_CBC); - REGISTER(CRYPTO_BLF_CBC); - REGISTER(CRYPTO_CAST_CBC); - REGISTER(CRYPTO_SKIPJACK_CBC); - REGISTER(CRYPTO_NULL_CBC); - REGISTER(CRYPTO_MD5_HMAC); - REGISTER(CRYPTO_SHA1_HMAC); - REGISTER(CRYPTO_SHA2_224_HMAC); - REGISTER(CRYPTO_SHA2_256_HMAC); - REGISTER(CRYPTO_SHA2_384_HMAC); - REGISTER(CRYPTO_SHA2_512_HMAC); - REGISTER(CRYPTO_RIPEMD160_HMAC); - REGISTER(CRYPTO_NULL_HMAC); - REGISTER(CRYPTO_MD5_KPDK); - REGISTER(CRYPTO_SHA1_KPDK); - REGISTER(CRYPTO_MD5); - REGISTER(CRYPTO_SHA1); - REGISTER(CRYPTO_SHA2_224); - REGISTER(CRYPTO_SHA2_256); - REGISTER(CRYPTO_SHA2_384); - REGISTER(CRYPTO_SHA2_512); - REGISTER(CRYPTO_RIJNDAEL128_CBC); - REGISTER(CRYPTO_AES_XTS); - REGISTER(CRYPTO_AES_ICM); - REGISTER(CRYPTO_AES_NIST_GCM_16); - REGISTER(CRYPTO_AES_NIST_GMAC); - REGISTER(CRYPTO_AES_128_NIST_GMAC); - REGISTER(CRYPTO_AES_192_NIST_GMAC); - REGISTER(CRYPTO_AES_256_NIST_GMAC); - REGISTER(CRYPTO_CAMELLIA_CBC); - REGISTER(CRYPTO_DEFLATE_COMP); - REGISTER(CRYPTO_BLAKE2B); - REGISTER(CRYPTO_BLAKE2S); - REGISTER(CRYPTO_CHACHA20); - REGISTER(CRYPTO_AES_CCM_16); - REGISTER(CRYPTO_AES_CCM_CBC_MAC); - REGISTER(CRYPTO_POLY1305); -#undef REGISTER - return 0; + return (0); } static int @@ -1390,6 +1490,7 @@ static device_method_t swcr_methods[] = { DEVMETHOD(device_attach, swcr_attach), DEVMETHOD(device_detach, swcr_detach), + DEVMETHOD(cryptodev_probesession, swcr_probesession), DEVMETHOD(cryptodev_newsession, swcr_newsession), DEVMETHOD(cryptodev_freesession,swcr_freesession), DEVMETHOD(cryptodev_process, swcr_process), diff --git a/sys/opencrypto/cryptosoft.h b/sys/opencrypto/cryptosoft.h deleted file mode 100644 index d787dc243ae6..000000000000 --- a/sys/opencrypto/cryptosoft.h +++ /dev/null @@ -1,71 +0,0 @@ -/* $FreeBSD$ */ -/* $OpenBSD: cryptosoft.h,v 1.10 2002/04/22 23:10:09 deraadt Exp $ */ - -/*- - * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) - * - * This code was written by Angelos D. Keromytis in Athens, Greece, in - * February 2000. Network Security Technologies Inc. (NSTI) kindly - * supported the development of this code. - * - * Copyright (c) 2000 Angelos D. Keromytis - * - * Permission to use, copy, and modify this software with or without fee - * is hereby granted, provided that this entire notice is included in - * all source code copies of any software which is or includes a copy or - * modification of this software. - * - * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR - * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY - * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE - * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR - * PURPOSE. - */ - -#ifndef _CRYPTO_CRYPTOSOFT_H_ -#define _CRYPTO_CRYPTOSOFT_H_ - -/* Software session entry */ -struct swcr_data { - int sw_alg; /* Algorithm */ - union { - struct { - u_int8_t *SW_ictx; - u_int8_t *SW_octx; - u_int16_t SW_klen; - u_int16_t SW_mlen; - struct auth_hash *SW_axf; - } SWCR_AUTH; - struct { - u_int8_t *SW_kschedule; - struct enc_xform *SW_exf; - } SWCR_ENC; - struct { - u_int32_t SW_size; - struct comp_algo *SW_cxf; - } SWCR_COMP; - } SWCR_UN; - -#define sw_ictx SWCR_UN.SWCR_AUTH.SW_ictx -#define sw_octx SWCR_UN.SWCR_AUTH.SW_octx -#define sw_klen SWCR_UN.SWCR_AUTH.SW_klen -#define sw_mlen SWCR_UN.SWCR_AUTH.SW_mlen -#define sw_axf SWCR_UN.SWCR_AUTH.SW_axf -#define sw_kschedule SWCR_UN.SWCR_ENC.SW_kschedule -#define sw_exf SWCR_UN.SWCR_ENC.SW_exf -#define sw_size SWCR_UN.SWCR_COMP.SW_size -#define sw_cxf SWCR_UN.SWCR_COMP.SW_cxf -}; - -struct swcr_session { - struct mtx swcr_lock; - struct swcr_data swcr_algorithms[2]; - unsigned swcr_nalgs; -}; - -#ifdef _KERNEL -extern u_int8_t hmac_ipad_buffer[]; -extern u_int8_t hmac_opad_buffer[]; -#endif /* _KERNEL */ - -#endif /* _CRYPTO_CRYPTO_H_ */ diff --git a/sys/opencrypto/ktls_ocf.c b/sys/opencrypto/ktls_ocf.c index 0f04e1268e2c..b607f2eead3d 100644 --- a/sys/opencrypto/ktls_ocf.c +++ b/sys/opencrypto/ktls_ocf.c @@ -45,7 +45,6 @@ __FBSDID("$FreeBSD$"); struct ocf_session { crypto_session_t sid; - int crda_alg; struct mtx lock; }; @@ -100,8 +99,6 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, { struct uio uio; struct tls_aead_data ad; - struct tls_nonce_data nd; - struct cryptodesc *crde, *crda; struct cryptop *crp; struct ocf_session *os; struct ocf_operation *oo; @@ -116,19 +113,15 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, oo->os = os; iov = oo->iov; - crp = crypto_getreq(2); - if (crp == NULL) { - free(oo, M_KTLS_OCF); - return (ENOMEM); - } + crp = crypto_getreq(os->sid, M_WAITOK); /* Setup the IV. */ - memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN); - memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); + memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN); + memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); /* Setup the AAD. */ tls_comp_len = ntohs(hdr->tls_length) - - (AES_GMAC_HASH_LEN + sizeof(nd.seq)); + (AES_GMAC_HASH_LEN + sizeof(uint64_t)); ad.seq = htobe64(seqno); ad.type = hdr->tls_type; ad.tls_vmajor = hdr->tls_vmajor; @@ -160,26 +153,20 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls, uio.uio_segflg = UIO_SYSSPACE; uio.uio_td = curthread; - crp->crp_session = os->sid; - crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; + crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; + crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; + crp->crp_buf_type = CRYPTO_BUF_UIO; crp->crp_uio = &uio; crp->crp_ilen = uio.uio_resid; crp->crp_opaque = oo; crp->crp_callback = ktls_ocf_callback; - crde = crp->crp_desc; - crda = crde->crd_next; - - crda->crd_alg = os->crda_alg; - crda->crd_skip = 0; - crda->crd_len = sizeof(ad); - crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; - - crde->crd_alg = CRYPTO_AES_NIST_GCM_16; - crde->crd_skip = sizeof(ad); - crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); - crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; - memcpy(crde->crd_iv, &nd, sizeof(nd)); + crp->crp_aad_start = 0; + crp->crp_aad_length = sizeof(ad); + crp->crp_payload_start = sizeof(ad); + crp->crp_payload_length = crp->crp_ilen - + (sizeof(ad) + AES_GMAC_HASH_LEN); + crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN; counter_u64_add(ocf_tls12_gcm_crypts, 1); for (;;) { @@ -216,7 +203,6 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, struct uio uio; struct tls_aead_data_13 ad; char nonce[12]; - struct cryptodesc *crde, *crda; struct cryptop *crp; struct ocf_session *os; struct ocf_operation *oo; @@ -230,11 +216,7 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, oo->os = os; iov = oo->iov; - crp = crypto_getreq(2); - if (crp == NULL) { - free(oo, M_KTLS_OCF); - return (ENOMEM); - } + crp = crypto_getreq(os->sid, M_WAITOK); /* Setup the nonce. */ memcpy(nonce, tls->params.iv, tls->params.iv_len); @@ -272,26 +254,21 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls, uio.uio_segflg = UIO_SYSSPACE; uio.uio_td = curthread; - crp->crp_session = os->sid; - crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM; + crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; + crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; + crp->crp_buf_type = CRYPTO_BUF_UIO; crp->crp_uio = &uio; crp->crp_ilen = uio.uio_resid; crp->crp_opaque = oo; crp->crp_callback = ktls_ocf_callback; - crde = crp->crp_desc; - crda = crde->crd_next; - - crda->crd_alg = os->crda_alg; - crda->crd_skip = 0; - crda->crd_len = sizeof(ad); - crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN; - - crde->crd_alg = CRYPTO_AES_NIST_GCM_16; - crde->crd_skip = sizeof(ad); - crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN); - crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; - memcpy(crde->crd_iv, nonce, sizeof(nonce)); + crp->crp_aad_start = 0; + crp->crp_aad_length = sizeof(ad); + crp->crp_payload_start = sizeof(ad); + crp->crp_payload_length = crp->crp_ilen - + (sizeof(ad) + AES_GMAC_HASH_LEN); + crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN; + memcpy(crp->crp_iv, nonce, sizeof(nonce)); counter_u64_add(ocf_tls13_gcm_crypts, 1); for (;;) { @@ -326,6 +303,7 @@ ktls_ocf_free(struct ktls_session *tls) struct ocf_session *os; os = tls->cipher; + crypto_freesession(os->sid); mtx_destroy(&os->lock); explicit_bzero(os, sizeof(*os)); free(os, M_KTLS_OCF); @@ -334,27 +312,26 @@ ktls_ocf_free(struct ktls_session *tls) static int ktls_ocf_try(struct socket *so, struct ktls_session *tls) { - struct cryptoini cria, crie; + struct crypto_session_params csp; struct ocf_session *os; int error; - memset(&cria, 0, sizeof(cria)); - memset(&crie, 0, sizeof(crie)); + memset(&csp, 0, sizeof(csp)); switch (tls->params.cipher_algorithm) { case CRYPTO_AES_NIST_GCM_16: switch (tls->params.cipher_key_len) { case 128 / 8: - cria.cri_alg = CRYPTO_AES_128_NIST_GMAC; - break; case 256 / 8: - cria.cri_alg = CRYPTO_AES_256_NIST_GMAC; break; default: return (EINVAL); } - cria.cri_key = tls->params.cipher_key; - cria.cri_klen = tls->params.cipher_key_len * 8; + csp.csp_mode = CSP_MODE_AEAD; + csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; + csp.csp_cipher_key = tls->params.cipher_key; + csp.csp_cipher_klen = tls->params.cipher_key_len; + csp.csp_ivlen = AES_GCM_IV_LEN; break; default: return (EPROTONOSUPPORT); @@ -370,19 +347,13 @@ ktls_ocf_try(struct socket *so, struct ktls_session *tls) if (os == NULL) return (ENOMEM); - crie.cri_alg = tls->params.cipher_algorithm; - crie.cri_key = tls->params.cipher_key; - crie.cri_klen = tls->params.cipher_key_len * 8; - - crie.cri_next = &cria; - error = crypto_newsession(&os->sid, &crie, + error = crypto_newsession(&os->sid, &csp, CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE); if (error) { free(os, M_KTLS_OCF); return (error); } - os->crda_alg = cria.cri_alg; mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); tls->cipher = os; if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) diff --git a/sys/opencrypto/xform_gmac.c b/sys/opencrypto/xform_gmac.c index 156ed7f13443..7359e014e5d5 100644 --- a/sys/opencrypto/xform_gmac.c +++ b/sys/opencrypto/xform_gmac.c @@ -66,7 +66,7 @@ struct enc_xform enc_xform_aes_nist_gmac = { /* Authentication instances */ struct auth_hash auth_hash_nist_gmac_aes_128 = { - CRYPTO_AES_128_NIST_GMAC, "GMAC-AES-128", + CRYPTO_AES_NIST_GMAC, "GMAC-AES-128", AES_128_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx), GMAC_BLOCK_LEN, (void (*)(void *)) AES_GMAC_Init, @@ -77,7 +77,7 @@ struct auth_hash auth_hash_nist_gmac_aes_128 = { }; struct auth_hash auth_hash_nist_gmac_aes_192 = { - CRYPTO_AES_192_NIST_GMAC, "GMAC-AES-192", + CRYPTO_AES_NIST_GMAC, "GMAC-AES-192", AES_192_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx), GMAC_BLOCK_LEN, (void (*)(void *)) AES_GMAC_Init, @@ -88,7 +88,7 @@ struct auth_hash auth_hash_nist_gmac_aes_192 = { }; struct auth_hash auth_hash_nist_gmac_aes_256 = { - CRYPTO_AES_256_NIST_GMAC, "GMAC-AES-256", + CRYPTO_AES_NIST_GMAC, "GMAC-AES-256", AES_256_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx), GMAC_BLOCK_LEN, (void (*)(void *)) AES_GMAC_Init, |
