aboutsummaryrefslogtreecommitdiff
path: root/sys/dev
diff options
context:
space:
mode:
authorJohn Baldwin <jhb@FreeBSD.org>2020-03-27 18:25:23 +0000
committerJohn Baldwin <jhb@FreeBSD.org>2020-03-27 18:25:23 +0000
commitc03414326909ed7a740be3ba63fbbef01fe513a8 (patch)
tree9067f28738df03bb4b685773c52ba32517468212 /sys/dev
parent4d94781b4d9e03b8dbd6604d7e2280d342d3cf7e (diff)
Refactor driver and consumer interfaces for OCF (in-kernel crypto).
- The linked list of cryptoini structures used in session initialization is replaced with a new flat structure: struct crypto_session_params. This session includes a new mode to define how the other fields should be interpreted. Available modes include: - COMPRESS (for compression/decompression) - CIPHER (for simply encryption/decryption) - DIGEST (computing and verifying digests) - AEAD (combined auth and encryption such as AES-GCM and AES-CCM) - ETA (combined auth and encryption using encrypt-then-authenticate) Additional modes could be added in the future (e.g. if we wanted to support TLS MtE for AES-CBC in the kernel we could add a new mode for that. TLS modes might also affect how AAD is interpreted, etc.) The flat structure also includes the key lengths and algorithms as before. However, code doesn't have to walk the linked list and switch on the algorithm to determine which key is the auth key vs encryption key. The 'csp_auth_*' fields are always used for auth keys and settings and 'csp_cipher_*' for cipher. (Compression algorithms are stored in csp_cipher_alg.) - Drivers no longer register a list of supported algorithms. This doesn't quite work when you factor in modes (e.g. a driver might support both AES-CBC and SHA2-256-HMAC separately but not combined for ETA). Instead, a new 'crypto_probesession' method has been added to the kobj interface for symmteric crypto drivers. This method returns a negative value on success (similar to how device_probe works) and the crypto framework uses this value to pick the "best" driver. There are three constants for hardware (e.g. ccr), accelerated software (e.g. aesni), and plain software (cryptosoft) that give preference in that order. One effect of this is that if you request only hardware when creating a new session, you will no longer get a session using accelerated software. Another effect is that the default setting to disallow software crypto via /dev/crypto now disables accelerated software. Once a driver is chosen, 'crypto_newsession' is invoked as before. - Crypto operations are now solely described by the flat 'cryptop' structure. The linked list of descriptors has been removed. A separate enum has been added to describe the type of data buffer in use instead of using CRYPTO_F_* flags to make it easier to add more types in the future if needed (e.g. wired userspace buffers for zero-copy). It will also make it easier to re-introduce separate input and output buffers (in-kernel TLS would benefit from this). Try to make the flags related to IV handling less insane: - CRYPTO_F_IV_SEPARATE means that the IV is stored in the 'crp_iv' member of the operation structure. If this flag is not set, the IV is stored in the data buffer at the 'crp_iv_start' offset. - CRYPTO_F_IV_GENERATE means that a random IV should be generated and stored into the data buffer. This cannot be used with CRYPTO_F_IV_SEPARATE. If a consumer wants to deal with explicit vs implicit IVs, etc. it can always generate the IV however it needs and store partial IVs in the buffer and the full IV/nonce in crp_iv and set CRYPTO_F_IV_SEPARATE. The layout of the buffer is now described via fields in cryptop. crp_aad_start and crp_aad_length define the boundaries of any AAD. Previously with GCM and CCM you defined an auth crd with this range, but for ETA your auth crd had to span both the AAD and plaintext (and they had to be adjacent). crp_payload_start and crp_payload_length define the boundaries of the plaintext/ciphertext. Modes that only do a single operation (COMPRESS, CIPHER, DIGEST) should only use this region and leave the AAD region empty. If a digest is present (or should be generated), it's starting location is marked by crp_digest_start. Instead of using the CRD_F_ENCRYPT flag to determine the direction of the operation, cryptop now includes an 'op' field defining the operation to perform. For digests I've added a new VERIFY digest mode which assumes a digest is present in the input and fails the request with EBADMSG if it doesn't match the internally-computed digest. GCM and CCM already assumed this, and the new AEAD mode requires this for decryption. The new ETA mode now also requires this for decryption, so IPsec and GELI no longer do their own authentication verification. Simple DIGEST operations can also do this, though there are no in-tree consumers. To eventually support some refcounting to close races, the session cookie is now passed to crypto_getop() and clients should no longer set crp_sesssion directly. - Assymteric crypto operation structures should be allocated via crypto_getkreq() and freed via crypto_freekreq(). This permits the crypto layer to track open asym requests and close races with a driver trying to unregister while asym requests are in flight. - crypto_copyback, crypto_copydata, crypto_apply, and crypto_contiguous_subsegment now accept the 'crp' object as the first parameter instead of individual members. This makes it easier to deal with different buffer types in the future as well as separate input and output buffers. It's also simpler for driver writers to use. - bus_dmamap_load_crp() loads a DMA mapping for a crypto buffer. This understands the various types of buffers so that drivers that use DMA do not have to be aware of different buffer types. - Helper routines now exist to build an auth context for HMAC IPAD and OPAD. This reduces some duplicated work among drivers. - Key buffers are now treated as const throughout the framework and in device drivers. However, session key buffers provided when a session is created are expected to remain alive for the duration of the session. - GCM and CCM sessions now only specify a cipher algorithm and a cipher key. The redundant auth information is not needed or used. - For cryptosoft, split up the code a bit such that the 'process' callback now invokes a function pointer in the session. This function pointer is set based on the mode (in effect) though it simplifies a few edge cases that would otherwise be in the switch in 'process'. It does split up GCM vs CCM which I think is more readable even if there is some duplication. - I changed /dev/crypto to support GMAC requests using CRYPTO_AES_NIST_GMAC as an auth algorithm and updated cryptocheck to work with it. - Combined cipher and auth sessions via /dev/crypto now always use ETA mode. The COP_F_CIPHER_FIRST flag is now a no-op that is ignored. This was actually documented as being true in crypto(4) before, but the code had not implemented this before I added the CIPHER_FIRST flag. - I have not yet updated /dev/crypto to be aware of explicit modes for sessions. I will probably do that at some point in the future as well as teach it about IV/nonce and tag lengths for AEAD so we can support all of the NIST KAT tests for GCM and CCM. - I've split up the exising crypto.9 manpage into several pages of which many are written from scratch. - I have converted all drivers and consumers in the tree and verified that they compile, but I have not tested all of them. I have tested the following drivers: - cryptosoft - aesni (AES only) - blake2 - ccr and the following consumers: - cryptodev - IPsec - ktls_ocf - GELI (lightly) I have not tested the following: - ccp - aesni with sha - hifn - kgssapi_krb5 - ubsec - padlock - safe - armv8_crypto (aarch64) - glxsb (i386) - sec (ppc) - cesa (armv7) - cryptocteon (mips64) - nlmsec (mips64) Discussed with: cem Relnotes: yes Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D23677
Notes
svn path=/head/; revision=359374
Diffstat (limited to 'sys/dev')
-rw-r--r--sys/dev/cesa/cesa.c599
-rw-r--r--sys/dev/cesa/cesa.h5
-rw-r--r--sys/dev/cxgbe/adapter.h2
-rw-r--r--sys/dev/cxgbe/crypto/t4_crypto.c1299
-rw-r--r--sys/dev/cxgbe/crypto/t4_keyctx.c40
-rw-r--r--sys/dev/cxgbe/tom/t4_tls.c4
-rw-r--r--sys/dev/glxsb/glxsb.c268
-rw-r--r--sys/dev/glxsb/glxsb.h6
-rw-r--r--sys/dev/glxsb/glxsb_hash.c100
-rw-r--r--sys/dev/hifn/hifn7751.c622
-rw-r--r--sys/dev/hifn/hifn7751var.h15
-rw-r--r--sys/dev/safe/safe.c915
-rw-r--r--sys/dev/safe/safevar.h12
-rw-r--r--sys/dev/sec/sec.c540
-rw-r--r--sys/dev/sec/sec.h18
-rw-r--r--sys/dev/ubsec/ubsec.c585
-rw-r--r--sys/dev/ubsec/ubsecvar.h10
17 files changed, 2191 insertions, 2849 deletions
diff --git a/sys/dev/cesa/cesa.c b/sys/dev/cesa/cesa.c
index 6cbac049bb42..d4e056d2a09b 100644
--- a/sys/dev/cesa/cesa.c
+++ b/sys/dev/cesa/cesa.c
@@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$");
#include <crypto/sha2/sha256.h>
#include <crypto/rijndael/rijndael.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform.h>
#include "cryptodev_if.h"
#include <arm/mv/mvreg.h>
@@ -80,7 +81,10 @@ static int cesa_attach(device_t);
static int cesa_attach_late(device_t);
static int cesa_detach(device_t);
static void cesa_intr(void *);
-static int cesa_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int cesa_probesession(device_t,
+ const struct crypto_session_params *);
+static int cesa_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int cesa_process(device_t, struct cryptop *, int);
static struct resource_spec cesa_res_spec[] = {
@@ -97,6 +101,7 @@ static device_method_t cesa_methods[] = {
DEVMETHOD(device_detach, cesa_detach),
/* Crypto device methods */
+ DEVMETHOD(cryptodev_probesession, cesa_probesession),
DEVMETHOD(cryptodev_newsession, cesa_newsession),
DEVMETHOD(cryptodev_process, cesa_process),
@@ -417,78 +422,68 @@ cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
return (0);
}
-static int
+static void
cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
{
- uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
- uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN];
- SHA1_CTX sha1ctx;
- SHA256_CTX sha256ctx;
- MD5_CTX md5ctx;
+ union authctx auth_ctx;
uint32_t *hout;
uint32_t *hin;
int i;
- memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
- memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
- for (i = 0; i < mklen; i++) {
- ipad[i] ^= mkey[i];
- opad[i] ^= mkey[i];
- }
-
hin = (uint32_t *)cs->cs_hiv_in;
hout = (uint32_t *)cs->cs_hiv_out;
switch (alg) {
case CRYPTO_MD5_HMAC:
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, ipad, MD5_BLOCK_LEN);
- memcpy(hin, md5ctx.state, sizeof(md5ctx.state));
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, opad, MD5_BLOCK_LEN);
- memcpy(hout, md5ctx.state, sizeof(md5ctx.state));
+ hmac_init_ipad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx);
+ memcpy(hin, auth_ctx.md5ctx.state,
+ sizeof(auth_ctx.md5ctx.state));
+ hmac_init_opad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx);
+ memcpy(hout, auth_ctx.md5ctx.state,
+ sizeof(auth_ctx.md5ctx.state));
break;
case CRYPTO_SHA1_HMAC:
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, ipad, SHA1_BLOCK_LEN);
- memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, opad, SHA1_BLOCK_LEN);
- memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
+ hmac_init_ipad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
+ memcpy(hin, auth_ctx.sha1ctx.h.b32,
+ sizeof(auth_ctx.sha1ctx.h.b32));
+ hmac_init_opad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
+ memcpy(hout, auth_ctx.sha1ctx.h.b32,
+ sizeof(auth_ctx.sha1ctx.h.b32));
break;
case CRYPTO_SHA2_256_HMAC:
- SHA256_Init(&sha256ctx);
- SHA256_Update(&sha256ctx, ipad, SHA2_256_BLOCK_LEN);
- memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state));
- SHA256_Init(&sha256ctx);
- SHA256_Update(&sha256ctx, opad, SHA2_256_BLOCK_LEN);
- memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state));
+ hmac_init_ipad(&auth_hash_hmac_sha2_256, mkey, mklen,
+ &auth_ctx);
+ memcpy(hin, auth_ctx.sha256ctx.state,
+ sizeof(auth_ctx.sha256ctx.state));
+ hmac_init_opad(&auth_hash_hmac_sha2_256, mkey, mklen,
+ &auth_ctx);
+ memcpy(hout, auth_ctx.sha256ctx.state,
+ sizeof(auth_ctx.sha256ctx.state));
break;
default:
- return (EINVAL);
+ panic("shouldn't get here");
}
for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
hin[i] = htobe32(hin[i]);
hout[i] = htobe32(hout[i]);
}
-
- return (0);
}
static int
-cesa_prep_aes_key(struct cesa_session *cs)
+cesa_prep_aes_key(struct cesa_session *cs,
+ const struct crypto_session_params *csp)
{
uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
uint32_t *dkey;
int i;
- rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8);
+ rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8);
cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
dkey = (uint32_t *)cs->cs_aes_dkey;
- switch (cs->cs_klen) {
+ switch (csp->csp_cipher_klen) {
case 16:
cs->cs_config |= CESA_CSH_AES_KLEN_128;
for (i = 0; i < 4; i++)
@@ -515,22 +510,6 @@ cesa_prep_aes_key(struct cesa_session *cs)
return (0);
}
-static int
-cesa_is_hash(int alg)
-{
-
- switch (alg) {
- case CRYPTO_MD5:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- return (1);
- default:
- return (0);
- }
-}
-
static void
cesa_start_packet(struct cesa_packet *cp, unsigned int size)
{
@@ -584,6 +563,7 @@ cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
unsigned int skip, len;
struct cesa_sa_desc *csd;
struct cesa_request *cr;
+ struct cryptop *crp;
struct cesa_softc *sc;
struct cesa_packet cp;
bus_dma_segment_t seg;
@@ -593,73 +573,107 @@ cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
cci = arg;
sc = cci->cci_sc;
cr = cci->cci_cr;
+ crp = cr->cr_crp;
if (error) {
cci->cci_error = error;
return;
}
- elen = cci->cci_enc ? cci->cci_enc->crd_len : 0;
- eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0;
- mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0;
- mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
-
- if (elen && mlen &&
- ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
- (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
- (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
+ /*
+ * Only do a combined op if the AAD is adjacent to the payload
+ * and the AAD length is a multiple of the IV length. The
+ * checks against 'config' are to avoid recursing when the
+ * logic below invokes separate operations.
+ */
+ config = cci->cci_config;
+ if (((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC ||
+ (config & CESA_CSHD_OP_MASK) == CESA_CSHD_ENC_AND_MAC) &&
+ crp->crp_aad_length != 0 &&
+ (crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) {
/*
* Data alignment in the request does not meet CESA requiremnts
* for combined encryption/decryption and hashing. We have to
* split the request to separate operations and process them
* one by one.
*/
- config = cci->cci_config;
if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
config &= ~CESA_CSHD_OP_MASK;
cci->cci_config = config | CESA_CSHD_MAC;
- cci->cci_enc = NULL;
- cci->cci_mac = cr->cr_mac;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
cci->cci_config = config | CESA_CSHD_ENC;
- cci->cci_enc = cr->cr_enc;
- cci->cci_mac = NULL;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
} else {
config &= ~CESA_CSHD_OP_MASK;
cci->cci_config = config | CESA_CSHD_ENC;
- cci->cci_enc = cr->cr_enc;
- cci->cci_mac = NULL;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
cci->cci_config = config | CESA_CSHD_MAC;
- cci->cci_enc = NULL;
- cci->cci_mac = cr->cr_mac;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
}
return;
}
+ mskip = mlen = eskip = elen = 0;
+
+ if (crp->crp_aad_length == 0) {
+ skip = crp->crp_payload_start;
+ len = crp->crp_payload_length;
+ switch (config & CESA_CSHD_OP_MASK) {
+ case CESA_CSHD_ENC:
+ eskip = skip;
+ elen = len;
+ break;
+ case CESA_CSHD_MAC:
+ mskip = skip;
+ mlen = len;
+ break;
+ default:
+ eskip = skip;
+ elen = len;
+ mskip = skip;
+ mlen = len;
+ break;
+ }
+ } else {
+ /*
+ * For an encryption-only separate request, only
+ * process the payload. For combined requests and
+ * hash-only requests, process the entire region.
+ */
+ switch (config & CESA_CSHD_OP_MASK) {
+ case CESA_CSHD_ENC:
+ skip = crp->crp_payload_start;
+ len = crp->crp_payload_length;
+ eskip = skip;
+ elen = len;
+ break;
+ case CESA_CSHD_MAC:
+ skip = crp->crp_aad_start;
+ len = crp->crp_aad_length + crp->crp_payload_length;
+ mskip = skip;
+ mlen = len;
+ break;
+ default:
+ skip = crp->crp_aad_start;
+ len = crp->crp_aad_length + crp->crp_payload_length;
+ mskip = skip;
+ mlen = len;
+ eskip = crp->crp_payload_start;
+ elen = crp->crp_payload_length;
+ break;
+ }
+ }
+
tmlen = mlen;
fragmented = 0;
mpsize = CESA_MAX_PACKET_SIZE;
mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
- if (elen && mlen) {
- skip = MIN(eskip, mskip);
- len = MAX(elen + eskip, mlen + mskip) - skip;
- } else if (elen) {
- skip = eskip;
- len = elen;
- } else {
- skip = mskip;
- len = mlen;
- }
-
/* Start first packet in chain */
cesa_start_packet(&cp, MIN(mpsize, len));
@@ -777,16 +791,9 @@ cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
}
}
-static void
-cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
- bus_size_t size, int error)
-{
-
- cesa_create_chain_cb(arg, segs, nseg, error);
-}
-
static int
-cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
+cesa_create_chain(struct cesa_softc *sc,
+ const struct crypto_session_params *csp, struct cesa_request *cr)
{
struct cesa_chain_info cci;
struct cesa_tdma_desc *ctd;
@@ -797,17 +804,17 @@ cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
CESA_LOCK_ASSERT(sc, sessions);
/* Create request metadata */
- if (cr->cr_enc) {
- if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC &&
- (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+ if (csp->csp_cipher_klen != 0) {
+ if (csp->csp_cipher_alg == CRYPTO_AES_CBC &&
+ !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
- cr->cr_cs->cs_klen);
+ csp->csp_cipher_klen);
else
memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
- cr->cr_cs->cs_klen);
+ csp->csp_cipher_klen);
}
- if (cr->cr_mac) {
+ if (csp->csp_auth_klen != 0) {
memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
CESA_MAX_HASH_LEN);
memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
@@ -823,37 +830,30 @@ cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
/* Prepare SA configuration */
config = cr->cr_cs->cs_config;
- if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+ if (csp->csp_cipher_alg != 0 &&
+ !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
config |= CESA_CSHD_DECRYPT;
- if (cr->cr_enc && !cr->cr_mac)
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
config |= CESA_CSHD_ENC;
- if (!cr->cr_enc && cr->cr_mac)
+ break;
+ case CSP_MODE_DIGEST:
config |= CESA_CSHD_MAC;
- if (cr->cr_enc && cr->cr_mac)
+ break;
+ case CSP_MODE_ETA:
config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
CESA_CSHD_ENC_AND_MAC;
+ break;
+ }
/* Create data packets */
cci.cci_sc = sc;
cci.cci_cr = cr;
- cci.cci_enc = cr->cr_enc;
- cci.cci_mac = cr->cr_mac;
cci.cci_config = config;
cci.cci_error = 0;
- if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
- error = bus_dmamap_load_uio(sc->sc_data_dtag,
- cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf,
- cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
- else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF)
- error = bus_dmamap_load_mbuf(sc->sc_data_dtag,
- cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf,
- cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
- else
- error = bus_dmamap_load(sc->sc_data_dtag,
- cr->cr_dmap, cr->cr_crp->crp_buf,
- cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci,
- BUS_DMA_NOWAIT);
+ error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp,
+ cesa_create_chain_cb, &cci, BUS_DMA_NOWAIT);
if (!error)
cr->cr_dmap_loaded = 1;
@@ -1385,18 +1385,6 @@ cesa_attach_late(device_t dev)
goto err8;
}
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- if (sc->sc_soc_id == MV_DEV_88F6828 ||
- sc->sc_soc_id == MV_DEV_88F6820 ||
- sc->sc_soc_id == MV_DEV_88F6810)
- crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-
return (0);
err8:
for (i = 0; i < CESA_REQUESTS; i++)
@@ -1487,6 +1475,7 @@ cesa_intr(void *arg)
struct cesa_request *cr, *tmp;
struct cesa_softc *sc;
uint32_t ecr, icr;
+ uint8_t hash[HASH_MAX_LEN];
int blocked;
sc = arg;
@@ -1547,11 +1536,19 @@ cesa_intr(void *arg)
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
cr->cr_crp->crp_etype = sc->sc_error;
- if (cr->cr_mac)
- crypto_copyback(cr->cr_crp->crp_flags,
- cr->cr_crp->crp_buf, cr->cr_mac->crd_inject,
- cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
-
+ if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) {
+ if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(cr->cr_crp,
+ cr->cr_crp->crp_digest_start,
+ cr->cr_cs->cs_hlen, hash);
+ if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash,
+ cr->cr_cs->cs_hlen) != 0)
+ cr->cr_crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(cr->cr_crp,
+ cr->cr_crp->crp_digest_start,
+ cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
+ }
crypto_done(cr->cr_crp);
cesa_free_request(sc, cr);
}
@@ -1571,42 +1568,98 @@ cesa_intr(void *arg)
crypto_unblock(sc->sc_cid, blocked);
}
-static int
-cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+static bool
+cesa_cipher_supported(const struct crypto_session_params *csp)
{
- struct cesa_session *cs;
- struct cesa_softc *sc;
- struct cryptoini *enc;
- struct cryptoini *mac;
- int error;
-
- sc = device_get_softc(dev);
- enc = NULL;
- mac = NULL;
- error = 0;
- /* Check and parse input */
- if (cesa_is_hash(cri->cri_alg))
- mac = cri;
- else
- enc = cri;
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_DES_CBC:
+ if (csp->csp_ivlen != DES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_3DES_CBC:
+ if (csp->csp_ivlen != DES3_BLOCK_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
- cri = cri->cri_next;
+ if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN)
+ return (false);
- if (cri) {
- if (!enc && !cesa_is_hash(cri->cri_alg))
- enc = cri;
+ return (true);
+}
- if (!mac && cesa_is_hash(cri->cri_alg))
- mac = cri;
+static bool
+cesa_auth_supported(struct cesa_softc *sc,
+ const struct crypto_session_params *csp)
+{
- if (cri->cri_next || !(enc && mac))
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA2_256_HMAC:
+ if (!(sc->sc_soc_id == MV_DEV_88F6828 ||
+ sc->sc_soc_id == MV_DEV_88F6820 ||
+ sc->sc_soc_id == MV_DEV_88F6810))
+ return (false);
+ /* FALLTHROUGH */
+ case CRYPTO_MD5:
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA1_HMAC:
+ break;
+ default:
+ return (false);
+ }
+
+ if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN)
+ return (false);
+
+ return (true);
+}
+
+static int
+cesa_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct cesa_softc *sc;
+
+ sc = device_get_softc(dev);
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!cesa_auth_supported(sc, csp))
return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!cesa_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!cesa_auth_supported(sc, csp) ||
+ !cesa_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
}
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
- if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) ||
- (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN))
- return (E2BIG);
+static int
+cesa_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct cesa_session *cs;
+ struct cesa_softc *sc;
+ int error;
+
+ sc = device_get_softc(dev);
+ error = 0;
/* Allocate session */
cs = crypto_get_driver_session(cses);
@@ -1616,106 +1669,89 @@ cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
cs->cs_ivlen = 1;
cs->cs_mblen = 1;
- if (enc) {
- switch (enc->cri_alg) {
- case CRYPTO_AES_CBC:
- cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
- cs->cs_ivlen = AES_BLOCK_LEN;
- break;
- case CRYPTO_DES_CBC:
- cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
- cs->cs_ivlen = DES_BLOCK_LEN;
- break;
- case CRYPTO_3DES_CBC:
- cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
- CESA_CSHD_CBC;
- cs->cs_ivlen = DES3_BLOCK_LEN;
- break;
- default:
- error = EINVAL;
- break;
- }
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
+ cs->cs_ivlen = AES_BLOCK_LEN;
+ break;
+ case CRYPTO_DES_CBC:
+ cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
+ cs->cs_ivlen = DES_BLOCK_LEN;
+ break;
+ case CRYPTO_3DES_CBC:
+ cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
+ CESA_CSHD_CBC;
+ cs->cs_ivlen = DES3_BLOCK_LEN;
+ break;
}
- if (!error && mac) {
- switch (mac->cri_alg) {
- case CRYPTO_MD5:
- cs->cs_mblen = 1;
- cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_MD5;
- break;
- case CRYPTO_MD5_HMAC:
- cs->cs_mblen = MD5_BLOCK_LEN;
- cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_MD5_HMAC;
- if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
- cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
- break;
- case CRYPTO_SHA1:
- cs->cs_mblen = 1;
- cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_SHA1;
- break;
- case CRYPTO_SHA1_HMAC:
- cs->cs_mblen = SHA1_BLOCK_LEN;
- cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_SHA1_HMAC;
- if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
- cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
- break;
- case CRYPTO_SHA2_256_HMAC:
- cs->cs_mblen = SHA2_256_BLOCK_LEN;
- cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
- break;
- default:
- error = EINVAL;
- break;
- }
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5:
+ cs->cs_mblen = 1;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_MD5;
+ break;
+ case CRYPTO_MD5_HMAC:
+ cs->cs_mblen = MD5_BLOCK_LEN;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_MD5_HMAC;
+ if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
+ cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
+ break;
+ case CRYPTO_SHA1:
+ cs->cs_mblen = 1;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_SHA1;
+ break;
+ case CRYPTO_SHA1_HMAC:
+ cs->cs_mblen = SHA1_BLOCK_LEN;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_SHA1_HMAC;
+ if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
+ cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ cs->cs_mblen = SHA2_256_BLOCK_LEN;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
+ break;
}
/* Save cipher key */
- if (!error && enc && enc->cri_key) {
- cs->cs_klen = enc->cri_klen / 8;
- memcpy(cs->cs_key, enc->cri_key, cs->cs_klen);
- if (enc->cri_alg == CRYPTO_AES_CBC)
- error = cesa_prep_aes_key(cs);
+ if (csp->csp_cipher_key != NULL) {
+ memcpy(cs->cs_key, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+ if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
+ error = cesa_prep_aes_key(cs, csp);
}
/* Save digest key */
- if (!error && mac && mac->cri_key)
- error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key,
- mac->cri_klen / 8);
+ if (csp->csp_auth_key != NULL)
+ cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key,
+ csp->csp_auth_klen);
- if (error)
- return (error);
-
- return (0);
+ return (error);
}
static int
cesa_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct cesa_request *cr;
struct cesa_session *cs;
- struct cryptodesc *crd;
- struct cryptodesc *enc;
- struct cryptodesc *mac;
struct cesa_softc *sc;
int error;
sc = device_get_softc(dev);
- crd = crp->crp_desc;
- enc = NULL;
- mac = NULL;
error = 0;
cs = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
/* Check and parse input */
if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) {
@@ -1724,25 +1760,16 @@ cesa_process(device_t dev, struct cryptop *crp, int hint)
return (0);
}
- if (cesa_is_hash(crd->crd_alg))
- mac = crd;
- else
- enc = crd;
-
- crd = crd->crd_next;
-
- if (crd) {
- if (!enc && !cesa_is_hash(crd->crd_alg))
- enc = crd;
-
- if (!mac && cesa_is_hash(crd->crd_alg))
- mac = crd;
-
- if (crd->crd_next || !(enc && mac)) {
- crp->crp_etype = EINVAL;
- crypto_done(crp);
- return (0);
- }
+ /*
+ * For requests with AAD, only requests where the AAD is
+ * immediately adjacent to the payload are supported.
+ */
+ if (crp->crp_aad_length != 0 &&
+ (crp->crp_aad_start + crp->crp_aad_length) !=
+ crp->crp_payload_start) {
+ crp->crp_etype = EINVAL;
+ crypto_done(crp);
+ return (0);
}
/*
@@ -1759,51 +1786,37 @@ cesa_process(device_t dev, struct cryptop *crp, int hint)
/* Prepare request */
cr->cr_crp = crp;
- cr->cr_enc = enc;
- cr->cr_mac = mac;
cr->cr_cs = cs;
CESA_LOCK(sc, sessions);
cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(cr->cr_csd->csd_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+ cr->cr_csd->csd_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(cr->cr_csd->csd_iv, crp->crp_iv, csp->csp_ivlen);
else
- arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0);
-
- if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
- } else if (enc) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ cr->cr_csd->csd_iv);
}
- if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) {
- cs->cs_klen = enc->crd_klen / 8;
- memcpy(cs->cs_key, enc->crd_key, cs->cs_klen);
- if (enc->crd_alg == CRYPTO_AES_CBC)
- error = cesa_prep_aes_key(cs);
- } else
- error = E2BIG;
+ if (crp->crp_cipher_key != NULL) {
+ memcpy(cs->cs_key, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
+ if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
+ error = cesa_prep_aes_key(cs, csp);
}
- if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN)
- error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key,
- mac->crd_klen / 8);
- else
- error = E2BIG;
- }
+ if (!error && crp->crp_auth_key != NULL)
+ cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key,
+ csp->csp_auth_klen);
/* Convert request to chain of TDMA and SA descriptors */
if (!error)
- error = cesa_create_chain(sc, cr);
+ error = cesa_create_chain(sc, csp, cr);
cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
CESA_UNLOCK(sc, sessions);
diff --git a/sys/dev/cesa/cesa.h b/sys/dev/cesa/cesa.h
index 9fa35b89b18f..449f4ecce5b2 100644
--- a/sys/dev/cesa/cesa.h
+++ b/sys/dev/cesa/cesa.h
@@ -194,7 +194,6 @@ struct cesa_sa_desc {
struct cesa_session {
uint32_t cs_config;
- unsigned int cs_klen;
unsigned int cs_ivlen;
unsigned int cs_hlen;
unsigned int cs_mblen;
@@ -208,8 +207,6 @@ struct cesa_request {
struct cesa_sa_data *cr_csd;
bus_addr_t cr_csd_paddr;
struct cryptop *cr_crp;
- struct cryptodesc *cr_enc;
- struct cryptodesc *cr_mac;
struct cesa_session *cr_cs;
bus_dmamap_t cr_dmap;
int cr_dmap_loaded;
@@ -272,8 +269,6 @@ struct cesa_softc {
struct cesa_chain_info {
struct cesa_softc *cci_sc;
struct cesa_request *cci_cr;
- struct cryptodesc *cci_enc;
- struct cryptodesc *cci_mac;
uint32_t cci_config;
int cci_error;
};
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index fd5487b01179..1f3ccedca9b3 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -1204,7 +1204,7 @@ union authctx;
void t4_aes_getdeckey(void *, const void *, unsigned int);
void t4_copy_partial_hash(int, union authctx *, void *);
void t4_init_gmac_hash(const char *, int, char *);
-void t4_init_hmac_digest(struct auth_hash *, u_int, char *, int, char *);
+void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *);
#ifdef DEV_NETMAP
/* t4_netmap.c */
diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c
index 5b924125b6e4..5ab8048ece4d 100644
--- a/sys/dev/cxgbe/crypto/t4_crypto.c
+++ b/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -165,7 +165,7 @@ struct ccr_session_blkcipher {
struct ccr_session {
bool active;
int pending;
- enum { HASH, HMAC, BLKCIPHER, AUTHENC, GCM, CCM } mode;
+ enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode;
union {
struct ccr_session_hmac hmac;
struct ccr_session_gmac gmac;
@@ -208,8 +208,8 @@ struct ccr_softc {
uint64_t stats_blkcipher_decrypt;
uint64_t stats_hash;
uint64_t stats_hmac;
- uint64_t stats_authenc_encrypt;
- uint64_t stats_authenc_decrypt;
+ uint64_t stats_eta_encrypt;
+ uint64_t stats_eta_decrypt;
uint64_t stats_gcm_encrypt;
uint64_t stats_gcm_decrypt;
uint64_t stats_ccm_encrypt;
@@ -230,9 +230,9 @@ struct ccr_softc {
* Non-hash-only requests require a PHYS_DSGL that describes the
* location to store the results of the encryption or decryption
* operation. This SGL uses a different format (PHYS_DSGL) and should
- * exclude the crd_skip bytes at the start of the data as well as
- * any AAD or IV. For authenticated encryption requests it should
- * cover include the destination of the hash or tag.
+ * exclude the skip bytes at the start of the data as well as any AAD
+ * or IV. For authenticated encryption requests it should include the
+ * destination of the hash or tag.
*
* The input payload may either be supplied inline as immediate data,
* or via a standard ULP_TX SGL. This SGL should include AAD,
@@ -251,12 +251,19 @@ ccr_populate_sglist(struct sglist *sg, struct cryptop *crp)
int error;
sglist_reset(sg);
- if (crp->crp_flags & CRYPTO_F_IMBUF)
- error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf);
- else if (crp->crp_flags & CRYPTO_F_IOV)
- error = sglist_append_uio(sg, (struct uio *)crp->crp_buf);
- else
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ error = sglist_append_mbuf(sg, crp->crp_mbuf);
+ break;
+ case CRYPTO_BUF_UIO:
+ error = sglist_append_uio(sg, crp->crp_uio);
+ break;
+ case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
+ break;
+ default:
+ error = EINVAL;
+ }
return (error);
}
@@ -436,16 +443,13 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
struct chcr_wr *crwr;
struct wrqe *wr;
struct auth_hash *axf;
- struct cryptodesc *crd;
char *dst;
u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len;
u_int hmac_ctrl, imm_len, iopad_size;
int error, sgl_nsegs, sgl_len, use_opad;
- crd = crp->crp_desc;
-
/* Reject requests with too large of an input buffer. */
- if (crd->crd_len > MAX_REQUEST_SIZE)
+ if (crp->crp_payload_length > MAX_REQUEST_SIZE)
return (EFBIG);
axf = s->hmac.auth_hash;
@@ -471,19 +475,19 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
hash_size_in_response = axf->hashsize;
transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
- if (crd->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
imm_len = axf->blocksize;
sgl_nsegs = 0;
sgl_len = 0;
- } else if (ccr_use_imm_data(transhdr_len, crd->crd_len)) {
- imm_len = crd->crd_len;
+ } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) {
+ imm_len = crp->crp_payload_length;
sgl_nsegs = 0;
sgl_len = 0;
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crd->crd_skip, crd->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg;
@@ -512,8 +516,8 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
V_CPL_TX_SEC_PDU_IVINSRTOFST(0));
- crwr->sec_cpl.pldlen = htobe32(crd->crd_len == 0 ? axf->blocksize :
- crd->crd_len);
+ crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ?
+ axf->blocksize : crp->crp_payload_length);
crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0));
@@ -527,7 +531,8 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
V_SCMD_HMAC_CTRL(hmac_ctrl));
crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_LAST_FRAG(0) |
- V_SCMD_MORE_FRAGS(crd->crd_len == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1));
+ V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) |
+ V_SCMD_MAC_ONLY(1));
memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len);
@@ -540,14 +545,14 @@ ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1));
dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES;
- if (crd->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
dst[0] = 0x80;
if (s->mode == HMAC)
*(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) =
htobe64(axf->blocksize << 3);
} else if (imm_len != 0)
- crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, dst);
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -561,15 +566,20 @@ static int
ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
const struct cpl_fw6_pld *cpl, int error)
{
- struct cryptodesc *crd;
+ uint8_t hash[HASH_MAX_LEN];
- crd = crp->crp_desc;
- if (error == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- s->hmac.hash_len, (c_caddr_t)(cpl + 1));
- }
+ if (error)
+ return (error);
- return (error);
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len,
+ hash);
+ if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len,
+ (cpl + 1));
+ return (0);
}
static int
@@ -578,34 +588,31 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct chcr_wr *crwr;
struct wrqe *wr;
- struct cryptodesc *crd;
char *dst;
u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
- u_int imm_len;
+ u_int imm_len, iv_len;
int dsgl_nsegs, dsgl_len;
int sgl_nsegs, sgl_len;
int error;
- crd = crp->crp_desc;
-
- if (s->blkcipher.key_len == 0 || crd->crd_len == 0)
+ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0)
return (EINVAL);
- if (crd->crd_alg == CRYPTO_AES_CBC &&
- (crd->crd_len % AES_BLOCK_LEN) != 0)
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
+ (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
return (EINVAL);
/* Reject requests with too large of an input buffer. */
- if (crd->crd_len > MAX_REQUEST_SIZE)
+ if (crp->crp_payload_length > MAX_REQUEST_SIZE)
return (EFBIG);
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
sglist_reset(sc->sg_dsgl);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crd->crd_skip,
- crd->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
@@ -617,23 +624,28 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
kctx_len = roundup2(s->blkcipher.key_len, 16);
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
- if (ccr_use_imm_data(transhdr_len, crd->crd_len +
- s->blkcipher.iv_len)) {
- imm_len = crd->crd_len;
+ /* For AES-XTS we send a 16-byte IV in the work request. */
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+ iv_len = AES_BLOCK_LEN;
+ else
+ iv_len = s->blkcipher.iv_len;
+
+ if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) {
+ imm_len = crp->crp_payload_length;
sgl_nsegs = 0;
sgl_len = 0;
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crd->crd_skip, crd->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
}
- wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len +
+ wr_len = roundup2(transhdr_len, 16) + iv_len +
roundup2(imm_len, 16) + sgl_len;
if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG);
@@ -647,24 +659,20 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
/*
* Read the existing IV from the request or generate a random
- * one if none is provided. Optionally copy the generated IV
- * into the output buffer if requested.
+ * one if none is provided.
*/
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
- else
- arc4rand(iv, s->blkcipher.iv_len, 0);
- if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, s->blkcipher.iv_len, iv);
- } else {
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, s->blkcipher.iv_len, iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, s->blkcipher.iv_len, 0);
+ crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+
+ /* Zero the remainder of the IV for AES-XTS. */
+ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len);
ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
crp);
@@ -677,10 +685,10 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
- crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + crd->crd_len);
+ crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length);
crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
- V_CPL_TX_SEC_PDU_CIPHERSTART(s->blkcipher.iv_len + 1) |
+ V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) |
V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0));
@@ -693,7 +701,7 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
- V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+ V_SCMD_IV_SIZE(iv_len / 2) |
V_SCMD_NUM_IVS(0));
crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_IV_GEN_CTRL(0) |
@@ -701,24 +709,24 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
- switch (crd->crd_alg) {
- case CRYPTO_AES_CBC:
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ switch (s->blkcipher.cipher_mode) {
+ case SCMD_CIPH_MODE_AES_CBC:
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
else
memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_ICM:
+ case SCMD_CIPH_MODE_AES_CTR:
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_XTS:
+ case SCMD_CIPH_MODE_AES_XTS:
key_half = s->blkcipher.key_len / 2;
memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
key_half);
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key + key_half,
s->blkcipher.enckey, key_half);
else
@@ -730,11 +738,11 @@ ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
- memcpy(dst, iv, s->blkcipher.iv_len);
- dst += s->blkcipher.iv_len;
+ memcpy(dst, iv, iv_len);
+ dst += iv_len;
if (imm_len != 0)
- crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, dst);
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -775,8 +783,7 @@ ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize)
}
static int
-ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
{
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct chcr_wr *crwr;
@@ -784,9 +791,9 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
struct auth_hash *axf;
char *dst;
u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
- u_int hash_size_in_response, imm_len, iopad_size;
- u_int aad_start, aad_len, aad_stop;
- u_int auth_start, auth_stop, auth_insert;
+ u_int hash_size_in_response, imm_len, iopad_size, iv_len;
+ u_int aad_start, aad_stop;
+ u_int auth_insert;
u_int cipher_start, cipher_stop;
u_int hmac_ctrl, input_len;
int dsgl_nsegs, dsgl_len;
@@ -797,34 +804,24 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* If there is a need in the future, requests with an empty
* payload could be supported as HMAC-only requests.
*/
- if (s->blkcipher.key_len == 0 || crde->crd_len == 0)
+ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0)
return (EINVAL);
- if (crde->crd_alg == CRYPTO_AES_CBC &&
- (crde->crd_len % AES_BLOCK_LEN) != 0)
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
+ (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
return (EINVAL);
- /*
- * Compute the length of the AAD (data covered by the
- * authentication descriptor but not the encryption
- * descriptor). To simplify the logic, AAD is only permitted
- * before the cipher/plain text, not after. This is true of
- * all currently-generated requests.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
- return (EINVAL);
- if (crda->crd_skip < crde->crd_skip) {
- if (crda->crd_skip + crda->crd_len > crde->crd_skip)
- aad_len = (crde->crd_skip - crda->crd_skip);
- else
- aad_len = crda->crd_len;
- } else
- aad_len = 0;
- if (aad_len + s->blkcipher.iv_len > MAX_AAD_LEN)
+ /* For AES-XTS we send a 16-byte IV in the work request. */
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+ iv_len = AES_BLOCK_LEN;
+ else
+ iv_len = s->blkcipher.iv_len;
+
+ if (crp->crp_aad_length + iv_len > MAX_AAD_LEN)
return (EINVAL);
axf = s->hmac.auth_hash;
hash_size_in_response = s->hmac.hash_len;
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
@@ -839,26 +836,26 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* output buffer.
*/
if (op_type == CHCR_ENCRYPT_OP) {
- if (s->blkcipher.iv_len + aad_len + crde->crd_len +
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG);
} else {
- if (s->blkcipher.iv_len + aad_len + crde->crd_len >
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
MAX_REQUEST_SIZE)
return (EFBIG);
}
sglist_reset(sc->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0,
- s->blkcipher.iv_len + aad_len);
+ iv_len + crp->crp_aad_length);
if (error)
return (error);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
- crde->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_ENCRYPT_OP) {
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -888,7 +885,7 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* inside of the AAD region, so a second copy is always
* required.
*/
- input_len = aad_len + crde->crd_len;
+ input_len = crp->crp_aad_length + crp->crp_payload_length;
/*
* The firmware hangs if sent a request which is a
@@ -902,26 +899,27 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
return (EFBIG);
if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response;
- if (ccr_use_imm_data(transhdr_len, s->blkcipher.iv_len + input_len)) {
+
+ if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len;
sgl_nsegs = 0;
sgl_len = 0;
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
- if (aad_len != 0) {
+ if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_skip, aad_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error)
return (error);
}
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -934,37 +932,25 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* Auth-data that overlaps with the cipher region is placed in
* the auth section.
*/
- if (aad_len != 0) {
- aad_start = s->blkcipher.iv_len + 1;
- aad_stop = aad_start + aad_len - 1;
+ if (crp->crp_aad_length != 0) {
+ aad_start = iv_len + 1;
+ aad_stop = aad_start + crp->crp_aad_length - 1;
} else {
aad_start = 0;
aad_stop = 0;
}
- cipher_start = s->blkcipher.iv_len + aad_len + 1;
+ cipher_start = iv_len + crp->crp_aad_length + 1;
if (op_type == CHCR_DECRYPT_OP)
cipher_stop = hash_size_in_response;
else
cipher_stop = 0;
- if (aad_len == crda->crd_len) {
- auth_start = 0;
- auth_stop = 0;
- } else {
- if (aad_len != 0)
- auth_start = cipher_start;
- else
- auth_start = s->blkcipher.iv_len + crda->crd_skip -
- crde->crd_skip + 1;
- auth_stop = (crde->crd_skip + crde->crd_len) -
- (crda->crd_skip + crda->crd_len) + cipher_stop;
- }
if (op_type == CHCR_DECRYPT_OP)
auth_insert = hash_size_in_response;
else
auth_insert = 0;
- wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len +
- roundup2(imm_len, 16) + sgl_len;
+ wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
+ sgl_len;
if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG);
wr = alloc_wrqe(wr_len, sc->txq);
@@ -977,24 +963,20 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
/*
* Read the existing IV from the request or generate a random
- * one if none is provided. Optionally copy the generated IV
- * into the output buffer if requested.
+ * one if none is provided.
*/
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- arc4rand(iv, s->blkcipher.iv_len, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, s->blkcipher.iv_len, 0);
+ crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+
+ /* Zero the remainder of the IV for AES-XTS. */
+ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len);
ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len,
op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp);
@@ -1007,7 +989,7 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
- crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + input_len);
+ crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
@@ -1016,8 +998,8 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
- V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) |
- V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
+ V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
+ V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
@@ -1030,7 +1012,7 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
V_SCMD_HMAC_CTRL(hmac_ctrl) |
- V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+ V_SCMD_IV_SIZE(iv_len / 2) |
V_SCMD_NUM_IVS(0));
crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_IV_GEN_CTRL(0) |
@@ -1038,24 +1020,24 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
- switch (crde->crd_alg) {
- case CRYPTO_AES_CBC:
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ switch (s->blkcipher.cipher_mode) {
+ case SCMD_CIPH_MODE_AES_CBC:
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
else
memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_ICM:
+ case SCMD_CIPH_MODE_AES_CTR:
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_XTS:
+ case SCMD_CIPH_MODE_AES_XTS:
key_half = s->blkcipher.key_len / 2;
memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
key_half);
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key + key_half,
s->blkcipher.enckey, key_half);
else
@@ -1070,20 +1052,20 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
- memcpy(dst, iv, s->blkcipher.iv_len);
- dst += s->blkcipher.iv_len;
+ memcpy(dst, iv, iv_len);
+ dst += iv_len;
if (imm_len != 0) {
- if (aad_len != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_skip, aad_len, dst);
- dst += aad_len;
+ if (crp->crp_aad_length != 0) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, dst);
+ dst += crp->crp_aad_length;
}
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
- crde->crd_len, dst);
- dst += crde->crd_len;
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
+ dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_inject, hash_size_in_response, dst);
+ crypto_copydata(crp, crp->crp_digest_start,
+ hash_size_in_response, dst);
} else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -1094,38 +1076,19 @@ ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
}
static int
-ccr_authenc_done(struct ccr_softc *sc, struct ccr_session *s,
+ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s,
struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
{
- struct cryptodesc *crd;
/*
* The updated IV to permit chained requests is at
* cpl->data[2], but OCF doesn't permit chained requests.
- *
- * For a decryption request, the hardware may do a verification
- * of the HMAC which will fail if the existing HMAC isn't in the
- * buffer. If that happens, clear the error and copy the HMAC
- * from the CPL reply into the buffer.
- *
- * For encryption requests, crd should be the cipher request
- * which will have CRD_F_ENCRYPT set. For decryption
- * requests, crp_desc will be the HMAC request which should
- * not have this flag set.
*/
- crd = crp->crp_desc;
- if (error == EBADMSG && !CHK_PAD_ERR_BIT(be64toh(cpl->data[0])) &&
- !(crd->crd_flags & CRD_F_ENCRYPT)) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- s->hmac.hash_len, (c_caddr_t)(cpl + 1));
- error = 0;
- }
return (error);
}
static int
-ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
{
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct chcr_wr *crwr;
@@ -1146,21 +1109,14 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* The crypto engine doesn't handle GCM requests with an empty
* payload, so handle those in software instead.
*/
- if (crde->crd_len == 0)
+ if (crp->crp_payload_length == 0)
return (EMSGSIZE);
- /*
- * AAD is only permitted before the cipher/plain text, not
- * after.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
- return (EMSGSIZE);
-
- if (crda->crd_len + AES_BLOCK_LEN > MAX_AAD_LEN)
+ if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN)
return (EMSGSIZE);
hash_size_in_response = s->gmac.hash_len;
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
@@ -1187,6 +1143,12 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
iv_len = s->blkcipher.iv_len;
/*
+ * GCM requests should always provide an explicit IV.
+ */
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+ return (EINVAL);
+
+ /*
* The output buffer consists of the cipher text followed by
* the tag when encrypting. For decryption it only contains
* the plain text.
@@ -1196,25 +1158,26 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* output buffer.
*/
if (op_type == CHCR_ENCRYPT_OP) {
- if (iv_len + crda->crd_len + crde->crd_len +
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG);
} else {
- if (iv_len + crda->crd_len + crde->crd_len > MAX_REQUEST_SIZE)
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
+ MAX_REQUEST_SIZE)
return (EFBIG);
}
sglist_reset(sc->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
- crda->crd_len);
+ crp->crp_aad_length);
if (error)
return (error);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
- crde->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_ENCRYPT_OP) {
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1241,7 +1204,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* inside of the AAD region, so a second copy is always
* required.
*/
- input_len = crda->crd_len + crde->crd_len;
+ input_len = crp->crp_aad_length + crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response;
if (input_len > MAX_REQUEST_SIZE)
@@ -1253,19 +1216,19 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_skip, crda->crd_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error)
return (error);
}
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1273,14 +1236,14 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
}
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
aad_start = iv_len + 1;
- aad_stop = aad_start + crda->crd_len - 1;
+ aad_stop = aad_start + crp->crp_aad_length - 1;
} else {
aad_start = 0;
aad_stop = 0;
}
- cipher_start = iv_len + crda->crd_len + 1;
+ cipher_start = iv_len + crp->crp_aad_length + 1;
if (op_type == CHCR_DECRYPT_OP)
cipher_stop = hash_size_in_response;
else
@@ -1302,29 +1265,7 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
crwr = wrtod(wr);
memset(crwr, 0, wr_len);
- /*
- * Read the existing IV from the request or generate a random
- * one if none is provided. Optionally copy the generated IV
- * into the output buffer if requested.
- *
- * If the input IV is 12 bytes, append an explicit 4-byte
- * counter of 1.
- */
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- arc4rand(iv, s->blkcipher.iv_len, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- }
+ memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
if (s->blkcipher.iv_len == 12)
*(uint32_t *)&iv[12] = htobe32(1);
@@ -1343,13 +1284,12 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
/*
* NB: cipherstop is explicitly set to 0. On encrypt it
- * should normally be set to 0 anyway (as the encrypt crd ends
- * at the end of the input). However, for decrypt the cipher
- * ends before the tag in the AUTHENC case (and authstop is
- * set to stop before the tag), but for GCM the cipher still
- * runs to the end of the buffer. Not sure if this is
- * intentional or a firmware quirk, but it is required for
- * working tag validation with GCM decryption.
+ * should normally be set to 0 anyway. However, for decrypt
+ * the cipher ends before the tag in the ETA case (and
+ * authstop is set to stop before the tag), but for GCM the
+ * cipher still runs to the end of the buffer. Not sure if
+ * this is intentional or a firmware quirk, but it is required
+ * for working tag validation with GCM decryption.
*/
crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
@@ -1390,17 +1330,17 @@ ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
memcpy(dst, iv, iv_len);
dst += iv_len;
if (imm_len != 0) {
- if (crda->crd_len != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_skip, crda->crd_len, dst);
- dst += crda->crd_len;
+ if (crp->crp_aad_length != 0) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, dst);
+ dst += crp->crp_aad_length;
}
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
- crde->crd_len, dst);
- dst += crde->crd_len;
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
+ dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_inject, hash_size_in_response, dst);
+ crypto_copydata(crp, crp->crp_digest_start,
+ hash_size_in_response, dst);
} else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -1429,8 +1369,7 @@ ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s,
* performing the operation in software. Derived from swcr_authenc().
*/
static void
-ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp)
{
struct auth_hash *axf;
struct enc_xform *exf;
@@ -1478,30 +1417,19 @@ ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp,
* This assumes a 12-byte IV from the crp. See longer comment
* above in ccr_gcm() for more details.
*/
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, 12);
- else
- arc4rand(iv, 12, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, 12, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, 12);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, 12, iv);
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+ error = EINVAL;
+ goto out;
}
+ memcpy(iv, crp->crp_iv, 12);
*(uint32_t *)&iv[12] = htobe32(1);
axf->Reinit(auth_ctx, iv, sizeof(iv));
/* MAC the AAD. */
- for (i = 0; i < crda->crd_len; i += sizeof(block)) {
- len = imin(crda->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+ len = imin(crp->crp_aad_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_aad_start + i, len, block);
bzero(block + len, sizeof(block) - len);
axf->Update(auth_ctx, block, sizeof(block));
}
@@ -1509,16 +1437,15 @@ ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp,
exf->reinit(kschedule, iv);
/* Do encryption with MAC */
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i, len, block);
bzero(block + len, sizeof(block) - len);
- if (crde->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
exf->encrypt(kschedule, block);
axf->Update(auth_ctx, block, len);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ block);
} else {
axf->Update(auth_ctx, block, len);
}
@@ -1526,35 +1453,37 @@ ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp,
/* Length block. */
bzero(block, sizeof(block));
- ((uint32_t *)block)[1] = htobe32(crda->crd_len * 8);
- ((uint32_t *)block)[3] = htobe32(crde->crd_len * 8);
+ ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
+ ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8);
axf->Update(auth_ctx, block, sizeof(block));
/* Finalize MAC. */
axf->Final(digest, auth_ctx);
/* Inject or validate tag. */
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest), digest);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+ digest);
error = 0;
} else {
char digest2[GMAC_DIGEST_LEN];
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest2), digest2);
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+ digest2);
if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
error = 0;
/* Tag matches, decrypt data. */
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ for (i = 0; i < crp->crp_payload_length;
+ i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i,
+ sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i,
+ len, block);
bzero(block + len, sizeof(block) - len);
exf->decrypt(kschedule, block);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i,
+ len, block);
}
} else
error = EBADMSG;
@@ -1571,8 +1500,8 @@ out:
}
static void
-generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde,
- u_int hash_size_in_response, const char *iv, char *b0)
+generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response,
+ const char *iv, char *b0)
{
u_int i, payload_len;
@@ -1583,7 +1512,7 @@ generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde,
b0[0] |= (((hash_size_in_response - 2) / 2) << 3);
/* Store the payload length as a big-endian value. */
- payload_len = crde->crd_len;
+ payload_len = crp->crp_payload_length;
for (i = 0; i < iv[0]; i++) {
b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len;
payload_len >>= 8;
@@ -1595,15 +1524,14 @@ generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde,
* start of block 1. This only assumes a 16-bit AAD length
* since T6 doesn't support large AAD sizes.
*/
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
b0[0] |= (1 << 6);
- *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crda->crd_len);
+ *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length);
}
}
static int
-ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
{
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct ulptx_idata *idata;
@@ -1625,14 +1553,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* The crypto engine doesn't handle CCM requests with an empty
* payload, so handle those in software instead.
*/
- if (crde->crd_len == 0)
- return (EMSGSIZE);
-
- /*
- * AAD is only permitted before the cipher/plain text, not
- * after.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
+ if (crp->crp_payload_length == 0)
return (EMSGSIZE);
/*
@@ -1640,14 +1561,21 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* request.
*/
b0_len = CCM_B0_SIZE;
- if (crda->crd_len != 0)
+ if (crp->crp_aad_length != 0)
b0_len += CCM_AAD_FIELD_SIZE;
- aad_len = b0_len + crda->crd_len;
+ aad_len = b0_len + crp->crp_aad_length;
+
+ /*
+ * CCM requests should always provide an explicit IV (really
+ * the nonce).
+ */
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+ return (EINVAL);
/*
- * Always assume a 12 byte input IV for now since that is what
- * OCF always generates. The full IV in the work request is
- * 16 bytes.
+ * Always assume a 12 byte input nonce for now since that is
+ * what OCF always generates. The full IV in the work request
+ * is 16 bytes.
*/
iv_len = AES_BLOCK_LEN;
@@ -1655,7 +1583,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
return (EMSGSIZE);
hash_size_in_response = s->ccm_mac.hash_len;
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
@@ -1670,11 +1598,12 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* output buffer.
*/
if (op_type == CHCR_ENCRYPT_OP) {
- if (iv_len + aad_len + crde->crd_len + hash_size_in_response >
- MAX_REQUEST_SIZE)
+ if (iv_len + aad_len + crp->crp_payload_length +
+ hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG);
} else {
- if (iv_len + aad_len + crde->crd_len > MAX_REQUEST_SIZE)
+ if (iv_len + aad_len + crp->crp_payload_length >
+ MAX_REQUEST_SIZE)
return (EFBIG);
}
sglist_reset(sc->sg_dsgl);
@@ -1682,13 +1611,13 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
aad_len);
if (error)
return (error);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
- crde->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_ENCRYPT_OP) {
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1715,7 +1644,7 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
* inside of the AAD region, so a second copy is always
* required.
*/
- input_len = aad_len + crde->crd_len;
+ input_len = aad_len + crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response;
if (input_len > MAX_REQUEST_SIZE)
@@ -1729,19 +1658,19 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
imm_len = b0_len;
sglist_reset(sc->sg_ulptx);
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_skip, crda->crd_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error)
return (error);
}
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1774,27 +1703,12 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
memset(crwr, 0, wr_len);
/*
- * Read the nonce from the request or generate a random one if
- * none is provided. Use the nonce to generate the full IV
- * with the counter set to 0.
+ * Read the nonce from the request. Use the nonce to generate
+ * the full IV with the counter set to 0.
*/
memset(iv, 0, iv_len);
iv[0] = (15 - AES_CCM_IV_LEN) - 1;
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN);
- else
- arc4rand(iv + 1, AES_CCM_IV_LEN, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv + 1);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv + 1);
- }
+ memcpy(iv + 1, crp->crp_iv, AES_CCM_IV_LEN);
ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
crp);
@@ -1851,20 +1765,20 @@ ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len);
dst += iv_len;
- generate_ccm_b0(crda, crde, hash_size_in_response, iv, dst);
+ generate_ccm_b0(crp, hash_size_in_response, iv, dst);
if (sgl_nsegs == 0) {
dst += b0_len;
- if (crda->crd_len != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_skip, crda->crd_len, dst);
- dst += crda->crd_len;
+ if (crp->crp_aad_length != 0) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, dst);
+ dst += crp->crp_aad_length;
}
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
- crde->crd_len, dst);
- dst += crde->crd_len;
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
+ dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_inject, hash_size_in_response, dst);
+ crypto_copydata(crp, crp->crp_digest_start,
+ hash_size_in_response, dst);
} else {
dst += CCM_B0_SIZE;
if (b0_len > CCM_B0_SIZE) {
@@ -1911,8 +1825,7 @@ ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s,
* performing the operation in software. Derived from swcr_authenc().
*/
static void
-ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp)
{
struct auth_hash *axf;
struct enc_xform *exf;
@@ -1956,31 +1869,20 @@ ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp,
if (error)
goto out;
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN);
- else
- arc4rand(iv, AES_CCM_IV_LEN, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv);
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+ error = EINVAL;
+ goto out;
}
+ memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN);
- auth_ctx->aes_cbc_mac_ctx.authDataLength = crda->crd_len;
- auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crde->crd_len;
+ auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length;
+ auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length;
axf->Reinit(auth_ctx, iv, sizeof(iv));
/* MAC the AAD. */
- for (i = 0; i < crda->crd_len; i += sizeof(block)) {
- len = imin(crda->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+ len = imin(crp->crp_aad_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_aad_start + i, len, block);
bzero(block + len, sizeof(block) - len);
axf->Update(auth_ctx, block, sizeof(block));
}
@@ -1988,16 +1890,15 @@ ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp,
exf->reinit(kschedule, iv);
/* Do encryption/decryption with MAC */
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i, len, block);
bzero(block + len, sizeof(block) - len);
- if (crde->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
axf->Update(auth_ctx, block, len);
exf->encrypt(kschedule, block);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ block);
} else {
exf->decrypt(kschedule, block);
axf->Update(auth_ctx, block, len);
@@ -2008,28 +1909,30 @@ ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp,
axf->Final(digest, auth_ctx);
/* Inject or validate tag. */
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest), digest);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+ digest);
error = 0;
} else {
- char digest2[GMAC_DIGEST_LEN];
+ char digest2[AES_CBC_MAC_HASH_LEN];
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest2), digest2);
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+ digest2);
if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
error = 0;
/* Tag matches, decrypt data. */
exf->reinit(kschedule, iv);
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ for (i = 0; i < crp->crp_payload_length;
+ i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i,
+ sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i,
+ len, block);
bzero(block + len, sizeof(block) - len);
exf->decrypt(kschedule, block);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i,
+ len, block);
}
} else
error = EBADMSG;
@@ -2096,11 +1999,11 @@ ccr_sysctls(struct ccr_softc *sc)
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD,
&sc->stats_blkcipher_decrypt, 0,
"Cipher decryption requests submitted");
- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_encrypt", CTLFLAG_RD,
- &sc->stats_authenc_encrypt, 0,
+ SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD,
+ &sc->stats_eta_encrypt, 0,
"Combined AES+HMAC encryption requests submitted");
- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_decrypt", CTLFLAG_RD,
- &sc->stats_authenc_decrypt, 0,
+ SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD,
+ &sc->stats_eta_decrypt, 0,
"Combined AES+HMAC decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD,
&sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted");
@@ -2161,25 +2064,6 @@ ccr_attach(device_t dev)
sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
ccr_sysctls(sc);
- crypto_register(cid, CRYPTO_SHA1, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_224, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_256, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_384, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_512, 0, 0);
- crypto_register(cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_224_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_384_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_512_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(cid, CRYPTO_AES_ICM, 0, 0);
- crypto_register(cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
- crypto_register(cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_XTS, 0, 0);
- crypto_register(cid, CRYPTO_AES_CCM_16, 0, 0);
- crypto_register(cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
return (0);
}
@@ -2207,48 +2091,48 @@ ccr_detach(device_t dev)
}
static void
-ccr_init_hash_digest(struct ccr_session *s, int cri_alg)
+ccr_init_hash_digest(struct ccr_session *s)
{
union authctx auth_ctx;
struct auth_hash *axf;
axf = s->hmac.auth_hash;
axf->Init(&auth_ctx);
- t4_copy_partial_hash(cri_alg, &auth_ctx, s->hmac.pads);
+ t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads);
}
-static int
+static bool
ccr_aes_check_keylen(int alg, int klen)
{
- switch (klen) {
+ switch (klen * 8) {
case 128:
case 192:
if (alg == CRYPTO_AES_XTS)
- return (EINVAL);
+ return (false);
break;
case 256:
break;
case 512:
if (alg != CRYPTO_AES_XTS)
- return (EINVAL);
+ return (false);
break;
default:
- return (EINVAL);
+ return (false);
}
- return (0);
+ return (true);
}
static void
-ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen)
+ccr_aes_setkey(struct ccr_session *s, const void *key, int klen)
{
unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size;
unsigned int opad_present;
- if (alg == CRYPTO_AES_XTS)
- kbits = klen / 2;
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+ kbits = (klen / 2) * 8;
else
- kbits = klen;
+ kbits = klen * 8;
switch (kbits) {
case 128:
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
@@ -2263,18 +2147,18 @@ ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen)
panic("should not get here");
}
- s->blkcipher.key_len = klen / 8;
+ s->blkcipher.key_len = klen;
memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
- switch (alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_XTS:
+ switch (s->blkcipher.cipher_mode) {
+ case SCMD_CIPH_MODE_AES_CBC:
+ case SCMD_CIPH_MODE_AES_XTS:
t4_aes_getdeckey(s->blkcipher.deckey, key, kbits);
break;
}
kctx_len = roundup2(s->blkcipher.key_len, 16);
switch (s->mode) {
- case AUTHENC:
+ case ETA:
mk_size = s->hmac.mk_size;
opad_present = 1;
iopad_size = roundup2(s->hmac.partial_digest_len, 16);
@@ -2309,171 +2193,220 @@ ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen)
}
kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
- V_KEY_CONTEXT_DUAL_CK(alg == CRYPTO_AES_XTS) |
+ V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode ==
+ SCMD_CIPH_MODE_AES_XTS) |
V_KEY_CONTEXT_OPAD_PRESENT(opad_present) |
V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) |
V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1));
}
+static bool
+ccr_auth_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA2_224:
+ case CRYPTO_SHA2_256:
+ case CRYPTO_SHA2_384:
+ case CRYPTO_SHA2_512:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_224_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+ccr_cipher_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_ICM:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+ return (ccr_aes_check_keylen(csp->csp_cipher_alg,
+ csp->csp_cipher_klen));
+}
+
static int
-ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ccr_cipher_mode(const struct crypto_session_params *csp)
{
- struct ccr_softc *sc;
- struct ccr_session *s;
- struct auth_hash *auth_hash;
- struct cryptoini *c, *hash, *cipher;
- unsigned int auth_mode, cipher_mode, iv_len, mk_size;
- unsigned int partial_digest_len;
- int error;
- bool gcm_hash, hmac;
- if (cri == NULL)
- return (EINVAL);
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ return (SCMD_CIPH_MODE_AES_CBC);
+ case CRYPTO_AES_ICM:
+ return (SCMD_CIPH_MODE_AES_CTR);
+ case CRYPTO_AES_NIST_GCM_16:
+ return (SCMD_CIPH_MODE_AES_GCM);
+ case CRYPTO_AES_XTS:
+ return (SCMD_CIPH_MODE_AES_XTS);
+ case CRYPTO_AES_CCM_16:
+ return (SCMD_CIPH_MODE_AES_CCM);
+ default:
+ return (SCMD_CIPH_MODE_NOP);
+ }
+}
+
+static int
+ccr_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ unsigned int cipher_mode;
- gcm_hash = false;
- hmac = false;
- cipher = NULL;
- hash = NULL;
- auth_hash = NULL;
- auth_mode = SCMD_AUTH_MODE_NOP;
- cipher_mode = SCMD_CIPH_MODE_NOP;
- iv_len = 0;
- mk_size = 0;
- partial_digest_len = 0;
- for (c = cri; c != NULL; c = c->cri_next) {
- switch (c->cri_alg) {
- case CRYPTO_SHA1:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_384:
- case CRYPTO_SHA2_512:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- case CRYPTO_AES_CCM_CBC_MAC:
- if (hash)
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!ccr_auth_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!ccr_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (EINVAL);
+ if (csp->csp_auth_mlen < 0 ||
+ csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
return (EINVAL);
- hash = c;
- switch (c->cri_alg) {
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- auth_hash = &auth_hash_hmac_sha1;
- auth_mode = SCMD_AUTH_MODE_SHA1;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
- partial_digest_len = SHA1_HASH_LEN;
- break;
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_224_HMAC:
- auth_hash = &auth_hash_hmac_sha2_224;
- auth_mode = SCMD_AUTH_MODE_SHA224;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
- partial_digest_len = SHA2_256_HASH_LEN;
- break;
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_256_HMAC:
- auth_hash = &auth_hash_hmac_sha2_256;
- auth_mode = SCMD_AUTH_MODE_SHA256;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
- partial_digest_len = SHA2_256_HASH_LEN;
- break;
- case CRYPTO_SHA2_384:
- case CRYPTO_SHA2_384_HMAC:
- auth_hash = &auth_hash_hmac_sha2_384;
- auth_mode = SCMD_AUTH_MODE_SHA512_384;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
- partial_digest_len = SHA2_512_HASH_LEN;
- break;
- case CRYPTO_SHA2_512:
- case CRYPTO_SHA2_512_HMAC:
- auth_hash = &auth_hash_hmac_sha2_512;
- auth_mode = SCMD_AUTH_MODE_SHA512_512;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
- partial_digest_len = SHA2_512_HASH_LEN;
- break;
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- gcm_hash = true;
- auth_mode = SCMD_AUTH_MODE_GHASH;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
- break;
- case CRYPTO_AES_CCM_CBC_MAC:
- auth_mode = SCMD_AUTH_MODE_CBCMAC;
- break;
- }
- switch (c->cri_alg) {
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- hmac = true;
- break;
- }
break;
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_XTS:
case CRYPTO_AES_CCM_16:
- if (cipher)
+ if (csp->csp_ivlen != AES_CCM_IV_LEN)
+ return (EINVAL);
+ if (csp->csp_auth_mlen < 0 ||
+ csp->csp_auth_mlen > AES_CBC_MAC_HASH_LEN)
return (EINVAL);
- cipher = c;
- switch (c->cri_alg) {
- case CRYPTO_AES_CBC:
- cipher_mode = SCMD_CIPH_MODE_AES_CBC;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_ICM:
- cipher_mode = SCMD_CIPH_MODE_AES_CTR;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_NIST_GCM_16:
- cipher_mode = SCMD_CIPH_MODE_AES_GCM;
- iv_len = AES_GCM_IV_LEN;
- break;
- case CRYPTO_AES_XTS:
- cipher_mode = SCMD_CIPH_MODE_AES_XTS;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_CCM_16:
- cipher_mode = SCMD_CIPH_MODE_AES_CCM;
- iv_len = AES_CCM_IV_LEN;
- break;
- }
- if (c->cri_key != NULL) {
- error = ccr_aes_check_keylen(c->cri_alg,
- c->cri_klen);
- if (error)
- return (error);
- }
break;
default:
return (EINVAL);
}
- }
- if (gcm_hash != (cipher_mode == SCMD_CIPH_MODE_AES_GCM))
- return (EINVAL);
- if ((auth_mode == SCMD_AUTH_MODE_CBCMAC) !=
- (cipher_mode == SCMD_CIPH_MODE_AES_CCM))
- return (EINVAL);
- if (hash == NULL && cipher == NULL)
+ break;
+ case CSP_MODE_ETA:
+ if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
return (EINVAL);
- if (hash != NULL) {
- if (hmac || gcm_hash || auth_mode == SCMD_AUTH_MODE_CBCMAC) {
- if (hash->cri_key == NULL)
- return (EINVAL);
- } else {
- if (hash->cri_key != NULL)
- return (EINVAL);
- }
}
+ if (csp->csp_cipher_klen != 0) {
+ cipher_mode = ccr_cipher_mode(csp);
+ if (cipher_mode == SCMD_CIPH_MODE_NOP)
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+ccr_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct ccr_softc *sc;
+ struct ccr_session *s;
+ struct auth_hash *auth_hash;
+ unsigned int auth_mode, cipher_mode, mk_size;
+ unsigned int partial_digest_len;
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA1_HMAC:
+ auth_hash = &auth_hash_hmac_sha1;
+ auth_mode = SCMD_AUTH_MODE_SHA1;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
+ partial_digest_len = SHA1_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_224:
+ case CRYPTO_SHA2_224_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_224;
+ auth_mode = SCMD_AUTH_MODE_SHA224;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+ partial_digest_len = SHA2_256_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_256:
+ case CRYPTO_SHA2_256_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_256;
+ auth_mode = SCMD_AUTH_MODE_SHA256;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+ partial_digest_len = SHA2_256_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_384:
+ case CRYPTO_SHA2_384_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_384;
+ auth_mode = SCMD_AUTH_MODE_SHA512_384;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+ partial_digest_len = SHA2_512_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_512:
+ case CRYPTO_SHA2_512_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_512;
+ auth_mode = SCMD_AUTH_MODE_SHA512_512;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+ partial_digest_len = SHA2_512_HASH_LEN;
+ break;
+ default:
+ auth_hash = NULL;
+ auth_mode = SCMD_AUTH_MODE_NOP;
+ mk_size = 0;
+ partial_digest_len = 0;
+ break;
+ }
+
+ cipher_mode = ccr_cipher_mode(csp);
+
+#ifdef INVARIANTS
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ if (cipher_mode == SCMD_CIPH_MODE_NOP ||
+ cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
+ cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+ panic("invalid cipher algo");
+ break;
+ case CSP_MODE_DIGEST:
+ if (auth_mode == SCMD_AUTH_MODE_NOP)
+ panic("invalid auth algo");
+ break;
+ case CSP_MODE_AEAD:
+ if (cipher_mode != SCMD_CIPH_MODE_AES_GCM &&
+ cipher_mode != SCMD_CIPH_MODE_AES_CCM)
+ panic("invalid aead cipher algo");
+ if (auth_mode != SCMD_AUTH_MODE_NOP)
+ panic("invalid aead auth aglo");
+ break;
+ case CSP_MODE_ETA:
+ if (cipher_mode == SCMD_CIPH_MODE_NOP ||
+ cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
+ cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+ panic("invalid cipher algo");
+ if (auth_mode == SCMD_AUTH_MODE_NOP)
+ panic("invalid auth algo");
+ break;
+ default:
+ panic("invalid csp mode");
+ }
+#endif
+
sc = device_get_softc(dev);
/*
@@ -2493,54 +2426,61 @@ ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
s = crypto_get_driver_session(cses);
- if (gcm_hash)
- s->mode = GCM;
- else if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
- s->mode = CCM;
- else if (hash != NULL && cipher != NULL)
- s->mode = AUTHENC;
- else if (hash != NULL) {
- if (hmac)
+ switch (csp->csp_mode) {
+ case CSP_MODE_AEAD:
+ if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+ s->mode = CCM;
+ else
+ s->mode = GCM;
+ break;
+ case CSP_MODE_ETA:
+ s->mode = ETA;
+ break;
+ case CSP_MODE_DIGEST:
+ if (csp->csp_auth_klen != 0)
s->mode = HMAC;
else
s->mode = HASH;
- } else {
- MPASS(cipher != NULL);
+ break;
+ case CSP_MODE_CIPHER:
s->mode = BLKCIPHER;
+ break;
}
- if (gcm_hash) {
- if (hash->cri_mlen == 0)
+
+ if (s->mode == GCM) {
+ if (csp->csp_auth_mlen == 0)
s->gmac.hash_len = AES_GMAC_HASH_LEN;
else
- s->gmac.hash_len = hash->cri_mlen;
- t4_init_gmac_hash(hash->cri_key, hash->cri_klen,
+ s->gmac.hash_len = csp->csp_auth_mlen;
+ t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen,
s->gmac.ghash_h);
- } else if (auth_mode == SCMD_AUTH_MODE_CBCMAC) {
- if (hash->cri_mlen == 0)
+ } else if (s->mode == CCM) {
+ if (csp->csp_auth_mlen == 0)
s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN;
else
- s->ccm_mac.hash_len = hash->cri_mlen;
- } else if (hash != NULL) {
+ s->ccm_mac.hash_len = csp->csp_auth_mlen;
+ } else if (auth_mode != SCMD_AUTH_MODE_NOP) {
s->hmac.auth_hash = auth_hash;
s->hmac.auth_mode = auth_mode;
s->hmac.mk_size = mk_size;
s->hmac.partial_digest_len = partial_digest_len;
- if (hash->cri_mlen == 0)
+ if (csp->csp_auth_mlen == 0)
s->hmac.hash_len = auth_hash->hashsize;
else
- s->hmac.hash_len = hash->cri_mlen;
- if (hmac)
+ s->hmac.hash_len = csp->csp_auth_mlen;
+ if (csp->csp_auth_key != NULL)
t4_init_hmac_digest(auth_hash, partial_digest_len,
- hash->cri_key, hash->cri_klen, s->hmac.pads);
+ csp->csp_auth_key, csp->csp_auth_klen,
+ s->hmac.pads);
else
- ccr_init_hash_digest(s, hash->cri_alg);
+ ccr_init_hash_digest(s);
}
- if (cipher != NULL) {
+ if (cipher_mode != SCMD_CIPH_MODE_NOP) {
s->blkcipher.cipher_mode = cipher_mode;
- s->blkcipher.iv_len = iv_len;
- if (cipher->cri_key != NULL)
- ccr_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
- cipher->cri_klen);
+ s->blkcipher.iv_len = csp->csp_ivlen;
+ if (csp->csp_cipher_key != NULL)
+ ccr_aes_setkey(s, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
}
s->active = true;
@@ -2568,15 +2508,12 @@ ccr_freesession(device_t dev, crypto_session_t cses)
static int
ccr_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct ccr_softc *sc;
struct ccr_session *s;
- struct cryptodesc *crd, *crda, *crde;
int error;
- if (crp == NULL)
- return (EINVAL);
-
- crd = crp->crp_desc;
+ csp = crypto_get_params(crp->crp_session);
s = crypto_get_driver_session(crp->crp_session);
sc = device_get_softc(dev);
@@ -2594,141 +2531,82 @@ ccr_process(device_t dev, struct cryptop *crp, int hint)
sc->stats_hash++;
break;
case HMAC:
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
+ if (crp->crp_auth_key != NULL)
t4_init_hmac_digest(s->hmac.auth_hash,
- s->hmac.partial_digest_len, crd->crd_key,
- crd->crd_klen, s->hmac.pads);
+ s->hmac.partial_digest_len, crp->crp_auth_key,
+ csp->csp_auth_klen, s->hmac.pads);
error = ccr_hash(sc, s, crp);
if (error == 0)
sc->stats_hmac++;
break;
case BLKCIPHER:
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crd->crd_alg,
- crd->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crd->crd_alg, crd->crd_key,
- crd->crd_klen);
- }
+ if (crp->crp_cipher_key != NULL)
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
error = ccr_blkcipher(sc, s, crp);
if (error == 0) {
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_blkcipher_encrypt++;
else
sc->stats_blkcipher_decrypt++;
}
break;
- case AUTHENC:
- error = 0;
- switch (crd->crd_alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_XTS:
- /* Only encrypt-then-authenticate supported. */
- crde = crd;
- crda = crd->crd_next;
- if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
- error = EINVAL;
- break;
- }
- break;
- default:
- crda = crd;
- crde = crd->crd_next;
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- error = EINVAL;
- break;
- }
- break;
- }
- if (error)
- break;
- if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
+ case ETA:
+ if (crp->crp_auth_key != NULL)
t4_init_hmac_digest(s->hmac.auth_hash,
- s->hmac.partial_digest_len, crda->crd_key,
- crda->crd_klen, s->hmac.pads);
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
- }
- error = ccr_authenc(sc, s, crp, crda, crde);
+ s->hmac.partial_digest_len, crp->crp_auth_key,
+ csp->csp_auth_klen, s->hmac.pads);
+ if (crp->crp_cipher_key != NULL)
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
+ error = ccr_eta(sc, s, crp);
if (error == 0) {
- if (crde->crd_flags & CRD_F_ENCRYPT)
- sc->stats_authenc_encrypt++;
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ sc->stats_eta_encrypt++;
else
- sc->stats_authenc_decrypt++;
+ sc->stats_eta_decrypt++;
}
break;
case GCM:
- error = 0;
- if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
- crde = crd;
- crda = crd->crd_next;
- } else {
- crda = crd;
- crde = crd->crd_next;
- }
- if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
- t4_init_gmac_hash(crda->crd_key, crda->crd_klen,
- s->gmac.ghash_h);
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
+ if (crp->crp_cipher_key != NULL) {
+ t4_init_gmac_hash(crp->crp_cipher_key,
+ csp->csp_cipher_klen, s->gmac.ghash_h);
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
}
- if (crde->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
mtx_unlock(&sc->lock);
- ccr_gcm_soft(s, crp, crda, crde);
+ ccr_gcm_soft(s, crp);
return (0);
}
- error = ccr_gcm(sc, s, crp, crda, crde);
+ error = ccr_gcm(sc, s, crp);
if (error == EMSGSIZE) {
sc->stats_sw_fallback++;
mtx_unlock(&sc->lock);
- ccr_gcm_soft(s, crp, crda, crde);
+ ccr_gcm_soft(s, crp);
return (0);
}
if (error == 0) {
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_gcm_encrypt++;
else
sc->stats_gcm_decrypt++;
}
break;
case CCM:
- error = 0;
- if (crd->crd_alg == CRYPTO_AES_CCM_16) {
- crde = crd;
- crda = crd->crd_next;
- } else {
- crda = crd;
- crde = crd->crd_next;
- }
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
+ if (crp->crp_cipher_key != NULL) {
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
}
- error = ccr_ccm(sc, s, crp, crda, crde);
+ error = ccr_ccm(sc, s, crp);
if (error == EMSGSIZE) {
sc->stats_sw_fallback++;
mtx_unlock(&sc->lock);
- ccr_ccm_soft(s, crp, crda, crde);
+ ccr_ccm_soft(s, crp);
return (0);
}
if (error == 0) {
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_ccm_encrypt++;
else
sc->stats_ccm_decrypt++;
@@ -2789,8 +2667,8 @@ do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
case BLKCIPHER:
error = ccr_blkcipher_done(sc, s, crp, cpl, error);
break;
- case AUTHENC:
- error = ccr_authenc_done(sc, s, crp, cpl, error);
+ case ETA:
+ error = ccr_eta_done(sc, s, crp, cpl, error);
break;
case GCM:
error = ccr_gcm_done(sc, s, crp, cpl, error);
@@ -2835,6 +2713,7 @@ static device_method_t ccr_methods[] = {
DEVMETHOD(device_attach, ccr_attach),
DEVMETHOD(device_detach, ccr_detach),
+ DEVMETHOD(cryptodev_probesession, ccr_probesession),
DEVMETHOD(cryptodev_newsession, ccr_newsession),
DEVMETHOD(cryptodev_freesession, ccr_freesession),
DEVMETHOD(cryptodev_process, ccr_process),
diff --git a/sys/dev/cxgbe/crypto/t4_keyctx.c b/sys/dev/cxgbe/crypto/t4_keyctx.c
index bceade0ec810..0f034f1be334 100644
--- a/sys/dev/cxgbe/crypto/t4_keyctx.c
+++ b/sys/dev/cxgbe/crypto/t4_keyctx.c
@@ -73,7 +73,7 @@ t4_init_gmac_hash(const char *key, int klen, char *ghash)
uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)];
int rounds;
- rounds = rijndaelKeySetupEnc(keysched, key, klen);
+ rounds = rijndaelKeySetupEnc(keysched, key, klen * 8);
rijndaelEncrypt(keysched, rounds, zeroes, ghash);
}
@@ -118,45 +118,19 @@ t4_copy_partial_hash(int alg, union authctx *auth_ctx, void *dst)
void
t4_init_hmac_digest(struct auth_hash *axf, u_int partial_digest_len,
- char *key, int klen, char *dst)
+ const char *key, int klen, char *dst)
{
union authctx auth_ctx;
- char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN];
- u_int i;
-
- /*
- * If the key is larger than the block size, use the digest of
- * the key as the key instead.
- */
- klen /= 8;
- if (klen > axf->blocksize) {
- axf->Init(&auth_ctx);
- axf->Update(&auth_ctx, key, klen);
- axf->Final(ipad, &auth_ctx);
- klen = axf->hashsize;
- } else
- memcpy(ipad, key, klen);
- memset(ipad + klen, 0, axf->blocksize - klen);
- memcpy(opad, ipad, axf->blocksize);
-
- for (i = 0; i < axf->blocksize; i++) {
- ipad[i] ^= HMAC_IPAD_VAL;
- opad[i] ^= HMAC_OPAD_VAL;
- }
-
- /*
- * Hash the raw ipad and opad and store the partial results in
- * the key context.
- */
- axf->Init(&auth_ctx);
- axf->Update(&auth_ctx, ipad, axf->blocksize);
+ hmac_init_ipad(axf, key, klen, &auth_ctx);
t4_copy_partial_hash(axf->type, &auth_ctx, dst);
dst += roundup2(partial_digest_len, 16);
- axf->Init(&auth_ctx);
- axf->Update(&auth_ctx, opad, axf->blocksize);
+
+ hmac_init_opad(axf, key, klen, &auth_ctx);
t4_copy_partial_hash(axf->type, &auth_ctx, dst);
+
+ explicit_bzero(&auth_ctx, sizeof(auth_ctx));
}
/*
diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c
index 57bae811f0f6..a82edd29bc3f 100644
--- a/sys/dev/cxgbe/tom/t4_tls.c
+++ b/sys/dev/cxgbe/tom/t4_tls.c
@@ -892,7 +892,7 @@ init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx)
k_ctx->tx_key_info_size += GMAC_BLOCK_LEN;
memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE);
t4_init_gmac_hash(tls->params.cipher_key,
- tls->params.cipher_key_len * 8, hash);
+ tls->params.cipher_key_len, hash);
} else {
switch (tls->params.auth_algorithm) {
case CRYPTO_SHA1_HMAC:
@@ -920,7 +920,7 @@ init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx)
k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2;
k_ctx->mac_secret_size = mac_key_size;
t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
- tls->params.auth_key_len * 8, hash);
+ tls->params.auth_key_len, hash);
}
k_ctx->frag_size = tls->params.max_frame_len;
diff --git a/sys/dev/glxsb/glxsb.c b/sys/dev/glxsb/glxsb.c
index 4d89e7d6756a..0e80b1dba2aa 100644
--- a/sys/dev/glxsb/glxsb.c
+++ b/sys/dev/glxsb/glxsb.c
@@ -51,7 +51,6 @@ __FBSDID("$FreeBSD$");
#include <dev/pci/pcireg.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
#include <opencrypto/xform.h>
#include "cryptodev_if.h"
@@ -172,8 +171,6 @@ struct glxsb_dma_map {
struct glxsb_taskop {
struct glxsb_session *to_ses; /* crypto session */
struct cryptop *to_crp; /* cryptop to perfom */
- struct cryptodesc *to_enccrd; /* enccrd to perform */
- struct cryptodesc *to_maccrd; /* maccrd to perform */
};
struct glxsb_softc {
@@ -204,13 +201,16 @@ static void glxsb_dma_free(struct glxsb_softc *, struct glxsb_dma_map *);
static void glxsb_rnd(void *);
static int glxsb_crypto_setup(struct glxsb_softc *);
-static int glxsb_crypto_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int glxsb_crypto_probesession(device_t,
+ const struct crypto_session_params *);
+static int glxsb_crypto_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static void glxsb_crypto_freesession(device_t, crypto_session_t);
static int glxsb_aes(struct glxsb_softc *, uint32_t, uint32_t,
- uint32_t, void *, int, void *);
+ uint32_t, const void *, int, const void *);
-static int glxsb_crypto_encdec(struct cryptop *, struct cryptodesc *,
- struct glxsb_session *, struct glxsb_softc *);
+static int glxsb_crypto_encdec(struct cryptop *, struct glxsb_session *,
+ struct glxsb_softc *);
static void glxsb_crypto_task(void *, int);
static int glxsb_crypto_process(device_t, struct cryptop *, int);
@@ -222,6 +222,7 @@ static device_method_t glxsb_methods[] = {
DEVMETHOD(device_detach, glxsb_detach),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, glxsb_crypto_probesession),
DEVMETHOD(cryptodev_newsession, glxsb_crypto_newsession),
DEVMETHOD(cryptodev_freesession, glxsb_crypto_freesession),
DEVMETHOD(cryptodev_process, glxsb_crypto_process),
@@ -477,47 +478,24 @@ glxsb_crypto_setup(struct glxsb_softc *sc)
mtx_init(&sc->sc_task_mtx, "glxsb_crypto_mtx", NULL, MTX_DEF);
- if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0) != 0)
- goto crypto_fail;
-
return (0);
-
-crypto_fail:
- device_printf(sc->sc_dev, "cannot register crypto\n");
- crypto_unregister_all(sc->sc_cid);
- mtx_destroy(&sc->sc_task_mtx);
- return (ENOMEM);
}
static int
-glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
- struct cryptoini *cri)
+glxsb_crypto_probesession(device_t dev, const struct crypto_session_params *csp)
{
- struct glxsb_softc *sc = device_get_softc(dev);
- struct glxsb_session *ses;
- struct cryptoini *encini, *macini;
- int error;
- if (sc == NULL || cri == NULL)
+ if (csp->csp_flags != 0)
return (EINVAL);
- encini = macini = NULL;
- for (; cri != NULL; cri = cri->cri_next) {
- switch(cri->cri_alg) {
+ /*
+ * We only support HMAC algorithms to be able to work with
+ * ipsec(4), so if we are asked only for authentication without
+ * encryption, don't pretend we can accelerate it.
+ */
+ switch (csp->csp_mode) {
+ case CSP_MODE_ETA:
+ switch (csp->csp_auth_alg) {
case CRYPTO_NULL_HMAC:
case CRYPTO_MD5_HMAC:
case CRYPTO_SHA1_HMAC:
@@ -525,43 +503,42 @@ glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
- if (macini != NULL)
- return (EINVAL);
- macini = cri;
break;
+ default:
+ return (EINVAL);
+ }
+ /* FALLTHROUGH */
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
case CRYPTO_AES_CBC:
- if (encini != NULL)
+ if (csp->csp_cipher_klen * 8 != 128)
return (EINVAL);
- encini = cri;
break;
default:
return (EINVAL);
}
+ default:
+ return (EINVAL);
}
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
- /*
- * We only support HMAC algorithms to be able to work with
- * ipsec(4), so if we are asked only for authentication without
- * encryption, don't pretend we can accellerate it.
- */
- if (encini == NULL)
- return (EINVAL);
+static int
+glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct glxsb_softc *sc = device_get_softc(dev);
+ struct glxsb_session *ses;
+ int error;
ses = crypto_get_driver_session(cses);
- if (encini->cri_alg == CRYPTO_AES_CBC) {
- if (encini->cri_klen != 128) {
- glxsb_crypto_freesession(sc->sc_dev, cses);
- return (EINVAL);
- }
- arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
- ses->ses_klen = encini->cri_klen;
- /* Copy the key (Geode LX wants the primary key only) */
- bcopy(encini->cri_key, ses->ses_key, sizeof(ses->ses_key));
- }
+ /* Copy the key (Geode LX wants the primary key only) */
+ if (csp->csp_cipher_key != NULL)
+ bcopy(csp->csp_cipher_key, ses->ses_key, sizeof(ses->ses_key));
- if (macini != NULL) {
- error = glxsb_hash_setup(ses, macini);
+ if (csp->csp_auth_alg != 0) {
+ error = glxsb_hash_setup(ses, csp);
if (error != 0) {
glxsb_crypto_freesession(sc->sc_dev, cses);
return (error);
@@ -574,19 +551,15 @@ glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
static void
glxsb_crypto_freesession(device_t dev, crypto_session_t cses)
{
- struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses;
- if (sc == NULL)
- return;
-
ses = crypto_get_driver_session(cses);
glxsb_hash_free(ses);
}
static int
glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc,
- uint32_t pdst, void *key, int len, void *iv)
+ uint32_t pdst, const void *key, int len, const void *iv)
{
uint32_t status;
int i;
@@ -652,23 +625,24 @@ glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc,
}
static int
-glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
- struct glxsb_session *ses, struct glxsb_softc *sc)
+glxsb_crypto_encdec(struct cryptop *crp, struct glxsb_session *ses,
+ struct glxsb_softc *sc)
{
char *op_src, *op_dst;
+ const void *key;
uint32_t op_psrc, op_pdst;
- uint8_t op_iv[SB_AES_BLOCK_SIZE], *piv;
+ uint8_t op_iv[SB_AES_BLOCK_SIZE];
int error;
int len, tlen, xlen;
int offset;
uint32_t control;
- if (crd == NULL || (crd->crd_len % SB_AES_BLOCK_SIZE) != 0)
+ if ((crp->crp_payload_length % SB_AES_BLOCK_SIZE) != 0)
return (EINVAL);
/* How much of our buffer will we need to use? */
- xlen = crd->crd_len > GLXSB_MAX_AES_LEN ?
- GLXSB_MAX_AES_LEN : crd->crd_len;
+ xlen = crp->crp_payload_length > GLXSB_MAX_AES_LEN ?
+ GLXSB_MAX_AES_LEN : crp->crp_payload_length;
/*
* XXX Check if we can have input == output on Geode LX.
@@ -680,73 +654,57 @@ glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
op_psrc = sc->sc_dma.dma_paddr;
op_pdst = sc->sc_dma.dma_paddr + xlen;
- if (crd->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
control = SB_CTL_ENC;
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
- else
- bcopy(ses->ses_iv, op_iv, sizeof(op_iv));
-
- if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, sizeof(op_iv), op_iv);
- }
- } else {
+ else
control = SB_CTL_DEC;
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
- else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, sizeof(op_iv), op_iv);
- }
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(op_iv, sizeof(op_iv), 0);
+ crypto_copyback(crp, crp->crp_iv_start, sizeof(op_iv), op_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(op_iv, crp->crp_iv, sizeof(op_iv));
+ else
+ crypto_copydata(crp, crp->crp_iv_start, sizeof(op_iv), op_iv);
+
offset = 0;
- tlen = crd->crd_len;
- piv = op_iv;
+ tlen = crp->crp_payload_length;
+
+ if (crp->crp_cipher_key != NULL)
+ key = crp->crp_cipher_key;
+ else
+ key = ses->ses_key;
/* Process the data in GLXSB_MAX_AES_LEN chunks */
while (tlen > 0) {
len = (tlen > GLXSB_MAX_AES_LEN) ? GLXSB_MAX_AES_LEN : tlen;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + offset, len, op_src);
+ crypto_copydata(crp, crp->crp_payload_start + offset, len,
+ op_src);
glxsb_dma_pre_op(sc, &sc->sc_dma);
- error = glxsb_aes(sc, control, op_psrc, op_pdst, ses->ses_key,
- len, op_iv);
+ error = glxsb_aes(sc, control, op_psrc, op_pdst, key, len,
+ op_iv);
glxsb_dma_post_op(sc, &sc->sc_dma);
if (error != 0)
return (error);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + offset, len, op_dst);
+ crypto_copyback(crp, crp->crp_payload_start + offset, len,
+ op_dst);
offset += len;
tlen -= len;
- if (tlen <= 0) { /* Ideally, just == 0 */
- /* Finished - put the IV in session IV */
- piv = ses->ses_iv;
- }
-
/*
- * Copy out last block for use as next iteration/session IV.
- *
- * piv is set to op_iv[] before the loop starts, but is
- * set to ses->ses_iv if we're going to exit the loop this
- * time.
+ * Copy out last block for use as next iteration IV.
*/
- if (crd->crd_flags & CRD_F_ENCRYPT)
- bcopy(op_dst + len - sizeof(op_iv), piv, sizeof(op_iv));
- else {
- /* Decryption, only need this if another iteration */
- if (tlen > 0) {
- bcopy(op_src + len - sizeof(op_iv), piv,
- sizeof(op_iv));
- }
- }
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ bcopy(op_dst + len - sizeof(op_iv), op_iv,
+ sizeof(op_iv));
+ else
+ bcopy(op_src + len - sizeof(op_iv), op_iv,
+ sizeof(op_iv));
} /* while */
/* All AES processing has now been done. */
@@ -759,30 +717,31 @@ static void
glxsb_crypto_task(void *arg, int pending)
{
struct glxsb_softc *sc = arg;
+ const struct crypto_session_params *csp;
struct glxsb_session *ses;
struct cryptop *crp;
- struct cryptodesc *enccrd, *maccrd;
int error;
- maccrd = sc->sc_to.to_maccrd;
- enccrd = sc->sc_to.to_enccrd;
crp = sc->sc_to.to_crp;
ses = sc->sc_to.to_ses;
+ csp = crypto_get_params(crp->crp_session);
/* Perform data authentication if requested before encryption */
- if (maccrd != NULL && maccrd->crd_next == enccrd) {
- error = glxsb_hash_process(ses, maccrd, crp);
+ if (csp->csp_mode == CSP_MODE_ETA &&
+ !CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = glxsb_hash_process(ses, csp, crp);
if (error != 0)
goto out;
}
- error = glxsb_crypto_encdec(crp, enccrd, ses, sc);
+ error = glxsb_crypto_encdec(crp, ses, sc);
if (error != 0)
goto out;
/* Perform data authentication if requested after encryption */
- if (maccrd != NULL && enccrd->crd_next == maccrd) {
- error = glxsb_hash_process(ses, maccrd, crp);
+ if (csp->csp_mode == CSP_MODE_ETA &&
+ CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = glxsb_hash_process(ses, csp, crp);
if (error != 0)
goto out;
}
@@ -801,52 +760,6 @@ glxsb_crypto_process(device_t dev, struct cryptop *crp, int hint)
{
struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses;
- struct cryptodesc *crd, *enccrd, *maccrd;
- int error = 0;
-
- enccrd = maccrd = NULL;
-
- /* Sanity check. */
- if (crp == NULL)
- return (EINVAL);
-
- if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
- error = EINVAL;
- goto fail;
- }
-
- for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
- switch (crd->crd_alg) {
- case CRYPTO_NULL_HMAC:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- if (maccrd != NULL) {
- error = EINVAL;
- goto fail;
- }
- maccrd = crd;
- break;
- case CRYPTO_AES_CBC:
- if (enccrd != NULL) {
- error = EINVAL;
- goto fail;
- }
- enccrd = crd;
- break;
- default:
- error = EINVAL;
- goto fail;
- }
- }
-
- if (enccrd == NULL || enccrd->crd_len % AES_BLOCK_LEN != 0) {
- error = EINVAL;
- goto fail;
- }
ses = crypto_get_driver_session(crp->crp_session);
@@ -857,17 +770,10 @@ glxsb_crypto_process(device_t dev, struct cryptop *crp, int hint)
}
sc->sc_task_count++;
- sc->sc_to.to_maccrd = maccrd;
- sc->sc_to.to_enccrd = enccrd;
sc->sc_to.to_crp = crp;
sc->sc_to.to_ses = ses;
mtx_unlock(&sc->sc_task_mtx);
taskqueue_enqueue(sc->sc_tq, &sc->sc_cryptotask);
return(0);
-
-fail:
- crp->crp_etype = error;
- crypto_done(crp);
- return (error);
}
diff --git a/sys/dev/glxsb/glxsb.h b/sys/dev/glxsb/glxsb.h
index fe5128a744c6..27e5bb44709c 100644
--- a/sys/dev/glxsb/glxsb.h
+++ b/sys/dev/glxsb/glxsb.h
@@ -37,8 +37,6 @@
struct glxsb_session {
uint32_t ses_key[4]; /* key */
- uint8_t ses_iv[SB_AES_BLOCK_SIZE]; /* initialization vector */
- int ses_klen; /* key len */
struct auth_hash *ses_axf;
uint8_t *ses_ictx;
uint8_t *ses_octx;
@@ -46,10 +44,10 @@ struct glxsb_session {
};
int glxsb_hash_setup(struct glxsb_session *ses,
- struct cryptoini *macini);
+ const struct crypto_session_params *csp);
int glxsb_hash_process(struct glxsb_session *ses,
- struct cryptodesc *maccrd, struct cryptop *crp);
+ const struct crypto_session_params *csp, struct cryptop *crp);
void glxsb_hash_free(struct glxsb_session *ses);
diff --git a/sys/dev/glxsb/glxsb_hash.c b/sys/dev/glxsb/glxsb_hash.c
index c5c2028103f2..73d9896ffc01 100644
--- a/sys/dev/glxsb/glxsb_hash.c
+++ b/sys/dev/glxsb/glxsb_hash.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/malloc.h>
-#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
#include <opencrypto/xform.h>
#include "glxsb.h"
@@ -51,92 +50,66 @@ __FBSDID("$FreeBSD$");
MALLOC_DECLARE(M_GLXSB);
static void
-glxsb_hash_key_setup(struct glxsb_session *ses, caddr_t key, int klen)
+glxsb_hash_key_setup(struct glxsb_session *ses, const char *key, int klen)
{
struct auth_hash *axf;
- int i;
- klen /= 8;
axf = ses->ses_axf;
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
-
- axf->Init(ses->ses_ictx);
- axf->Update(ses->ses_ictx, key, klen);
- axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
- axf->Init(ses->ses_octx);
- axf->Update(ses->ses_octx, key, klen);
- axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ hmac_init_ipad(axf, key, klen, ses->ses_ictx);
+ hmac_init_opad(axf, key, klen, ses->ses_octx);
}
/*
* Compute keyed-hash authenticator.
*/
static int
-glxsb_authcompute(struct glxsb_session *ses, struct cryptodesc *crd,
- caddr_t buf, int flags)
+glxsb_authcompute(struct glxsb_session *ses, struct cryptop *crp)
{
- u_char hash[HASH_MAX_LEN];
+ u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN];
struct auth_hash *axf;
union authctx ctx;
int error;
axf = ses->ses_axf;
bcopy(ses->ses_ictx, &ctx, axf->ctxsize);
- error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+ error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
(int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
if (error != 0)
return (error);
+ error = crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length,
+ (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+ if (error != 0)
+ return (error);
+
axf->Final(hash, &ctx);
bcopy(ses->ses_octx, &ctx, axf->ctxsize);
axf->Update(&ctx, hash, axf->hashsize);
axf->Final(hash, &ctx);
- /* Inject the authentication data */
- crypto_copyback(flags, buf, crd->crd_inject,
- ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+ /* Verify or inject the authentication data */
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen,
+ hash2);
+ if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen,
+ hash);
return (0);
}
int
-glxsb_hash_setup(struct glxsb_session *ses, struct cryptoini *macini)
+glxsb_hash_setup(struct glxsb_session *ses,
+ const struct crypto_session_params *csp)
{
- ses->ses_mlen = macini->cri_mlen;
-
- /* Find software structure which describes HMAC algorithm. */
- switch (macini->cri_alg) {
- case CRYPTO_NULL_HMAC:
- ses->ses_axf = &auth_hash_null;
- break;
- case CRYPTO_MD5_HMAC:
- ses->ses_axf = &auth_hash_hmac_md5;
- break;
- case CRYPTO_SHA1_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha1;
- break;
- case CRYPTO_RIPEMD160_HMAC:
- ses->ses_axf = &auth_hash_hmac_ripemd_160;
- break;
- case CRYPTO_SHA2_256_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_256;
- break;
- case CRYPTO_SHA2_384_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_384;
- break;
- case CRYPTO_SHA2_512_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_512;
- break;
- }
+ ses->ses_axf = crypto_auth_hash(csp);
+ if (csp->csp_auth_mlen == 0)
+ ses->ses_mlen = ses->ses_axf->hashsize;
+ else
+ ses->ses_mlen = csp->csp_auth_mlen;
/* Allocate memory for HMAC inner and outer contexts. */
ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_GLXSB,
@@ -147,23 +120,24 @@ glxsb_hash_setup(struct glxsb_session *ses, struct cryptoini *macini)
return (ENOMEM);
/* Setup key if given. */
- if (macini->cri_key != NULL) {
- glxsb_hash_key_setup(ses, macini->cri_key,
- macini->cri_klen);
+ if (csp->csp_auth_key != NULL) {
+ glxsb_hash_key_setup(ses, csp->csp_auth_key,
+ csp->csp_auth_klen);
}
return (0);
}
int
-glxsb_hash_process(struct glxsb_session *ses, struct cryptodesc *maccrd,
- struct cryptop *crp)
+glxsb_hash_process(struct glxsb_session *ses,
+ const struct crypto_session_params *csp, struct cryptop *crp)
{
int error;
- if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
- glxsb_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+ if (crp->crp_auth_key != NULL)
+ glxsb_hash_key_setup(ses, crp->crp_auth_key,
+ csp->csp_auth_klen);
- error = glxsb_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+ error = glxsb_authcompute(ses, crp);
return (error);
}
diff --git a/sys/dev/hifn/hifn7751.c b/sys/dev/hifn/hifn7751.c
index ce0f060aa7d2..7f1889767090 100644
--- a/sys/dev/hifn/hifn7751.c
+++ b/sys/dev/hifn/hifn7751.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
+#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -71,6 +72,7 @@ __FBSDID("$FreeBSD$");
#include <sys/rman.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
#include <sys/random.h>
#include <sys/kobj.h>
@@ -102,7 +104,9 @@ static int hifn_suspend(device_t);
static int hifn_resume(device_t);
static int hifn_shutdown(device_t);
-static int hifn_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int hifn_probesession(device_t, const struct crypto_session_params *);
+static int hifn_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int hifn_process(device_t, struct cryptop *, int);
static device_method_t hifn_methods[] = {
@@ -115,6 +119,7 @@ static device_method_t hifn_methods[] = {
DEVMETHOD(device_shutdown, hifn_shutdown),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, hifn_probesession),
DEVMETHOD(cryptodev_newsession, hifn_newsession),
DEVMETHOD(cryptodev_process, hifn_process),
@@ -356,7 +361,7 @@ hifn_attach(device_t dev)
caddr_t kva;
int rseg, rid;
char rbase;
- u_int16_t ena, rev;
+ uint16_t rev;
sc->sc_dev = dev;
@@ -558,33 +563,22 @@ hifn_attach(device_t dev)
2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
printf("\n");
- sc->sc_cid = crypto_get_driverid(dev, sizeof(struct hifn_session),
- CRYPTOCAP_F_HARDWARE);
- if (sc->sc_cid < 0) {
- device_printf(dev, "could not get crypto driver id\n");
- goto fail_intr;
- }
-
WRITE_REG_0(sc, HIFN_0_PUCNFG,
READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
- ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
+ sc->sc_ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
- switch (ena) {
+ switch (sc->sc_ena) {
case HIFN_PUSTAT_ENA_2:
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
- if (sc->sc_flags & HIFN_HAS_AES)
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
- /*FALLTHROUGH*/
case HIFN_PUSTAT_ENA_1:
- crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
+ sc->sc_cid = crypto_get_driverid(dev,
+ sizeof(struct hifn_session), CRYPTOCAP_F_HARDWARE);
+ if (sc->sc_cid < 0) {
+ device_printf(dev, "could not get crypto driver id\n");
+ goto fail_intr;
+ }
break;
}
-
+
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
@@ -1547,6 +1541,7 @@ hifn_init_dma(struct hifn_softc *sc)
static u_int
hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
{
+ struct cryptop *crp;
u_int8_t *buf_pos;
hifn_base_command_t *base_cmd;
hifn_mac_command_t *mac_cmd;
@@ -1554,6 +1549,7 @@ hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
int using_mac, using_crypt, len, ivlen;
u_int32_t dlen, slen;
+ crp = cmd->crp;
buf_pos = buf;
using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
@@ -1576,24 +1572,27 @@ hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
if (using_mac) {
mac_cmd = (hifn_mac_command_t *)buf_pos;
- dlen = cmd->maccrd->crd_len;
+ dlen = crp->crp_aad_length + crp->crp_payload_length;
mac_cmd->source_count = htole16(dlen & 0xffff);
dlen >>= 16;
mac_cmd->masks = htole16(cmd->mac_masks |
((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
- mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
+ if (crp->crp_aad_length != 0)
+ mac_cmd->header_skip = htole16(crp->crp_aad_start);
+ else
+ mac_cmd->header_skip = htole16(crp->crp_payload_start);
mac_cmd->reserved = 0;
buf_pos += sizeof(hifn_mac_command_t);
}
if (using_crypt) {
cry_cmd = (hifn_crypt_command_t *)buf_pos;
- dlen = cmd->enccrd->crd_len;
+ dlen = crp->crp_payload_length;
cry_cmd->source_count = htole16(dlen & 0xffff);
dlen >>= 16;
cry_cmd->masks = htole16(cmd->cry_masks |
((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
- cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
+ cry_cmd->header_skip = htole16(crp->crp_payload_length);
cry_cmd->reserved = 0;
buf_pos += sizeof(hifn_crypt_command_t);
}
@@ -1782,15 +1781,30 @@ hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
return (idx);
}
+static bus_size_t
+hifn_crp_length(struct cryptop *crp)
+{
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (crp->crp_mbuf->m_pkthdr.len);
+ case CRYPTO_BUF_UIO:
+ return (crp->crp_uio->uio_resid);
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_ilen);
+ default:
+ panic("bad crp buffer type");
+ }
+}
+
static void
-hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, int error)
{
struct hifn_operand *op = arg;
KASSERT(nsegs <= MAX_SCATTER,
("hifn_op_cb: too many DMA segments (%u > %u) "
"returned when mapping operand", nsegs, MAX_SCATTER));
- op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
@@ -1832,130 +1846,110 @@ hifn_crypto(
return (ENOMEM);
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
- cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_load++;
- err = ENOMEM;
- goto err_srcmap1;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
- cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_load++;
- err = ENOMEM;
- goto err_srcmap1;
- }
- } else {
- err = EINVAL;
+ if (bus_dmamap_load_crp(sc->sc_dmat, cmd->src_map, crp, hifn_op_cb,
+ &cmd->src, BUS_DMA_NOWAIT)) {
+ hifnstats.hst_nomem_load++;
+ err = ENOMEM;
goto err_srcmap1;
}
+ cmd->src_mapsize = hifn_crp_length(crp);
if (hifn_dmamap_aligned(&cmd->src)) {
cmd->sloplen = cmd->src_mapsize & 3;
cmd->dst = cmd->src;
- } else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- err = EINVAL;
- goto err_srcmap;
- } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
- int totlen, len;
- struct mbuf *m, *m0, *mlast;
+ } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ int totlen, len;
+ struct mbuf *m, *m0, *mlast;
- KASSERT(cmd->dst_m == cmd->src_m,
- ("hifn_crypto: dst_m initialized improperly"));
- hifnstats.hst_unaligned++;
- /*
- * Source is not aligned on a longword boundary.
- * Copy the data to insure alignment. If we fail
- * to allocate mbufs or clusters while doing this
- * we return ERESTART so the operation is requeued
- * at the crypto later, but only if there are
- * ops already posted to the hardware; otherwise we
- * have no guarantee that we'll be re-entered.
- */
- totlen = cmd->src_mapsize;
- if (cmd->src_m->m_flags & M_PKTHDR) {
- len = MHLEN;
- MGETHDR(m0, M_NOWAIT, MT_DATA);
- if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
- m_free(m0);
- m0 = NULL;
- }
- } else {
- len = MLEN;
- MGET(m0, M_NOWAIT, MT_DATA);
+ KASSERT(cmd->dst_m == NULL,
+ ("hifn_crypto: dst_m initialized improperly"));
+ hifnstats.hst_unaligned++;
+
+ /*
+ * Source is not aligned on a longword boundary.
+ * Copy the data to insure alignment. If we fail
+ * to allocate mbufs or clusters while doing this
+ * we return ERESTART so the operation is requeued
+ * at the crypto later, but only if there are
+ * ops already posted to the hardware; otherwise we
+ * have no guarantee that we'll be re-entered.
+ */
+ totlen = cmd->src_mapsize;
+ if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+ len = MHLEN;
+ MGETHDR(m0, M_NOWAIT, MT_DATA);
+ if (m0 && !m_dup_pkthdr(m0, crp->crp_mbuf, M_NOWAIT)) {
+ m_free(m0);
+ m0 = NULL;
}
- if (m0 == NULL) {
+ } else {
+ len = MLEN;
+ MGET(m0, M_NOWAIT, MT_DATA);
+ }
+ if (m0 == NULL) {
+ hifnstats.hst_nomem_mbuf++;
+ err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ goto err_srcmap;
+ }
+ if (totlen >= MINCLSIZE) {
+ if (!(MCLGET(m0, M_NOWAIT))) {
+ hifnstats.hst_nomem_mcl++;
+ err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ m_freem(m0);
+ goto err_srcmap;
+ }
+ len = MCLBYTES;
+ }
+ totlen -= len;
+ m0->m_pkthdr.len = m0->m_len = len;
+ mlast = m0;
+
+ while (totlen > 0) {
+ MGET(m, M_NOWAIT, MT_DATA);
+ if (m == NULL) {
hifnstats.hst_nomem_mbuf++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ m_freem(m0);
goto err_srcmap;
}
+ len = MLEN;
if (totlen >= MINCLSIZE) {
- if (!(MCLGET(m0, M_NOWAIT))) {
+ if (!(MCLGET(m, M_NOWAIT))) {
hifnstats.hst_nomem_mcl++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ mlast->m_next = m;
m_freem(m0);
goto err_srcmap;
}
len = MCLBYTES;
}
- totlen -= len;
- m0->m_pkthdr.len = m0->m_len = len;
- mlast = m0;
- while (totlen > 0) {
- MGET(m, M_NOWAIT, MT_DATA);
- if (m == NULL) {
- hifnstats.hst_nomem_mbuf++;
- err = sc->sc_cmdu ? ERESTART : ENOMEM;
- m_freem(m0);
- goto err_srcmap;
- }
- len = MLEN;
- if (totlen >= MINCLSIZE) {
- if (!(MCLGET(m, M_NOWAIT))) {
- hifnstats.hst_nomem_mcl++;
- err = sc->sc_cmdu ? ERESTART : ENOMEM;
- mlast->m_next = m;
- m_freem(m0);
- goto err_srcmap;
- }
- len = MCLBYTES;
- }
-
- m->m_len = len;
- m0->m_pkthdr.len += len;
- totlen -= len;
+ m->m_len = len;
+ m0->m_pkthdr.len += len;
+ totlen -= len;
- mlast->m_next = m;
- mlast = m;
- }
- cmd->dst_m = m0;
+ mlast->m_next = m;
+ mlast = m;
}
- }
+ cmd->dst_m = m0;
- if (cmd->dst_map == NULL) {
- if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
+ if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
+ &cmd->dst_map)) {
hifnstats.hst_nomem_map++;
err = ENOMEM;
goto err_srcmap;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
- cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_map++;
- err = ENOMEM;
- goto err_dstmap1;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
- cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_load++;
- err = ENOMEM;
- goto err_dstmap1;
- }
+
+ if (bus_dmamap_load_mbuf_sg(sc->sc_dmat, cmd->dst_map, m0,
+ cmd->dst_segs, &cmd->dst_nsegs, 0)) {
+ hifnstats.hst_nomem_map++;
+ err = ENOMEM;
+ goto err_dstmap1;
}
+ cmd->dst_mapsize = m0->m_pkthdr.len;
+ } else {
+ err = EINVAL;
+ goto err_srcmap;
}
#ifdef HIFN_DEBUG
@@ -2111,8 +2105,8 @@ err_dstmap1:
if (cmd->src_map != cmd->dst_map)
bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
err_srcmap:
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (cmd->src_m != cmd->dst_m)
+ if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ if (cmd->dst_m != NULL)
m_freem(cmd->dst_m);
}
bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
@@ -2307,67 +2301,121 @@ hifn_intr(void *arg)
}
}
-/*
- * Allocate a new 'session' and return an encoded session id. 'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
- */
-static int
-hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+static bool
+hifn_auth_supported(struct hifn_softc *sc,
+ const struct crypto_session_params *csp)
{
- struct hifn_softc *sc = device_get_softc(dev);
- struct cryptoini *c;
- int mac = 0, cry = 0;
- struct hifn_session *ses;
- KASSERT(sc != NULL, ("hifn_newsession: null softc"));
- if (cri == NULL || sc == NULL)
- return (EINVAL);
+ switch (sc->sc_ena) {
+ case HIFN_PUSTAT_ENA_2:
+ case HIFN_PUSTAT_ENA_1:
+ break;
+ default:
+ return (false);
+ }
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5:
+ case CRYPTO_SHA1:
+ break;
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ if (csp->csp_auth_klen > HIFN_MAC_KEY_LENGTH)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
- ses = crypto_get_driver_session(cses);
+ return (true);
+}
- for (c = cri; c != NULL; c = c->cri_next) {
- switch (c->cri_alg) {
- case CRYPTO_MD5:
- case CRYPTO_SHA1:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- if (mac)
- return (EINVAL);
- mac = 1;
- ses->hs_mlen = c->cri_mlen;
- if (ses->hs_mlen == 0) {
- switch (c->cri_alg) {
- case CRYPTO_MD5:
- case CRYPTO_MD5_HMAC:
- ses->hs_mlen = 16;
- break;
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- ses->hs_mlen = 20;
- break;
- }
- }
- break;
- case CRYPTO_DES_CBC:
+static bool
+hifn_cipher_supported(struct hifn_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_cipher_klen == 0)
+ return (false);
+ if (csp->csp_ivlen > HIFN_MAX_IV_LENGTH)
+ return (false);
+ switch (sc->sc_ena) {
+ case HIFN_PUSTAT_ENA_2:
+ switch (csp->csp_cipher_alg) {
case CRYPTO_3DES_CBC:
- case CRYPTO_AES_CBC:
- /* XXX this may read fewer, does it matter? */
- read_random(ses->hs_iv,
- c->cri_alg == CRYPTO_AES_CBC ?
- HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
- /*FALLTHROUGH*/
case CRYPTO_ARC4:
- if (cry)
- return (EINVAL);
- cry = 1;
break;
- default:
- return (EINVAL);
+ case CRYPTO_AES_CBC:
+ if ((sc->sc_flags & HIFN_HAS_AES) == 0)
+ return (false);
+ switch (csp->csp_cipher_klen) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return (false);
+ }
+ return (true);
}
+ /*FALLTHROUGH*/
+ case HIFN_PUSTAT_ENA_1:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ return (true);
+ }
+ break;
}
- if (mac == 0 && cry == 0)
+ return (false);
+}
+
+static int
+hifn_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct hifn_softc *sc;
+
+ sc = device_get_softc(dev);
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!hifn_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!hifn_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!hifn_auth_supported(sc, csp) ||
+ !hifn_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ default:
return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+/*
+ * Allocate a new 'session'.
+ */
+static int
+hifn_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct hifn_session *ses;
+
+ ses = crypto_get_driver_session(cses);
+
+ if (csp->csp_auth_alg != 0) {
+ if (csp->csp_auth_mlen == 0)
+ ses->hs_mlen = crypto_auth_hash(csp)->hashsize;
+ else
+ ses->hs_mlen = csp->csp_auth_mlen;
+ }
+
return (0);
}
@@ -2379,18 +2427,15 @@ hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
static int
hifn_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct hifn_softc *sc = device_get_softc(dev);
struct hifn_command *cmd = NULL;
- int err, ivlen;
- struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
+ const void *mackey;
+ int err, ivlen, keylen;
struct hifn_session *ses;
- if (crp == NULL || crp->crp_callback == NULL) {
- hifnstats.hst_invalid++;
- return (EINVAL);
- }
-
ses = crypto_get_driver_session(crp->crp_session);
+
cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
if (cmd == NULL) {
hifnstats.hst_nomem++;
@@ -2398,80 +2443,26 @@ hifn_process(device_t dev, struct cryptop *crp, int hint)
goto errout;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- cmd->src_m = (struct mbuf *)crp->crp_buf;
- cmd->dst_m = (struct mbuf *)crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- cmd->src_io = (struct uio *)crp->crp_buf;
- cmd->dst_io = (struct uio *)crp->crp_buf;
- } else {
- err = EINVAL;
- goto errout; /* XXX we don't handle contiguous buffers! */
- }
+ csp = crypto_get_params(crp->crp_session);
- crd1 = crp->crp_desc;
- if (crd1 == NULL) {
+ /*
+ * The driver only supports ETA requests where there is no
+ * gap between the AAD and payload.
+ */
+ if (csp->csp_mode == CSP_MODE_ETA && crp->crp_aad_length != 0 &&
+ crp->crp_aad_start + crp->crp_aad_length !=
+ crp->crp_payload_start) {
err = EINVAL;
goto errout;
}
- crd2 = crd1->crd_next;
-
- if (crd2 == NULL) {
- if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1 ||
- crd1->crd_alg == CRYPTO_MD5) {
- maccrd = crd1;
- enccrd = NULL;
- } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_ARC4) {
- if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
- cmd->base_masks |= HIFN_BASE_CMD_DECODE;
- maccrd = NULL;
- enccrd = crd1;
- } else {
- err = EINVAL;
- goto errout;
- }
- } else {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_MD5 ||
- crd1->crd_alg == CRYPTO_SHA1) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC ||
- crd2->crd_alg == CRYPTO_AES_CBC ||
- crd2->crd_alg == CRYPTO_ARC4) &&
- ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
- cmd->base_masks = HIFN_BASE_CMD_DECODE;
- maccrd = crd1;
- enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_ARC4 ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC ||
- crd2->crd_alg == CRYPTO_MD5 ||
- crd2->crd_alg == CRYPTO_SHA1) &&
- (crd1->crd_flags & CRD_F_ENCRYPT)) {
- enccrd = crd1;
- maccrd = crd2;
- } else {
- /*
- * We cannot order the 7751 as requested
- */
- err = EINVAL;
- goto errout;
- }
- }
- if (enccrd) {
- cmd->enccrd = enccrd;
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ case CSP_MODE_ETA:
+ if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ cmd->base_masks |= HIFN_BASE_CMD_DECODE;
cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
- switch (enccrd->crd_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_ARC4:
cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
break;
@@ -2494,36 +2485,24 @@ hifn_process(device_t dev, struct cryptop *crp, int hint)
err = EINVAL;
goto errout;
}
- if (enccrd->crd_alg != CRYPTO_ARC4) {
- ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
- HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, cmd->iv, ivlen);
- else
- bcopy(ses->hs_iv, cmd->iv, ivlen);
-
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
- == 0) {
- crypto_copyback(crp->crp_flags,
- crp->crp_buf, enccrd->crd_inject,
- ivlen, cmd->iv);
- }
- } else {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, cmd->iv, ivlen);
- else {
- crypto_copydata(crp->crp_flags,
- crp->crp_buf, enccrd->crd_inject,
- ivlen, cmd->iv);
- }
- }
+ if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+ ivlen = csp->csp_ivlen;
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(cmd->iv, ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, ivlen,
+ cmd->iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(cmd->iv, crp->crp_iv, ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, ivlen,
+ cmd->iv);
}
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
- cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
- cmd->ck = enccrd->crd_key;
- cmd->cklen = enccrd->crd_klen >> 3;
+ if (crp->crp_cipher_key != NULL)
+ cmd->ck = crp->crp_cipher_key;
+ else
+ cmd->ck = csp->csp_cipher_key;
+ cmd->cklen = csp->csp_cipher_klen;
cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
/*
@@ -2546,13 +2525,15 @@ hifn_process(device_t dev, struct cryptop *crp, int hint)
goto errout;
}
}
+ break;
}
- if (maccrd) {
- cmd->maccrd = maccrd;
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ case CSP_MODE_ETA:
cmd->base_masks |= HIFN_BASE_CMD_MAC;
- switch (maccrd->crd_alg) {
+ switch (csp->csp_auth_alg) {
case CRYPTO_MD5:
cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
@@ -2575,12 +2556,16 @@ hifn_process(device_t dev, struct cryptop *crp, int hint)
break;
}
- if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
- maccrd->crd_alg == CRYPTO_MD5_HMAC) {
+ if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC ||
+ csp->csp_auth_alg == CRYPTO_MD5_HMAC) {
cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
- bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
- bzero(cmd->mac + (maccrd->crd_klen >> 3),
- HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
+ if (crp->crp_auth_key != NULL)
+ mackey = crp->crp_auth_key;
+ else
+ mackey = csp->csp_auth_key;
+ keylen = csp->csp_auth_klen;
+ bcopy(mackey, cmd->mac, keylen);
+ bzero(cmd->mac + keylen, HIFN_MAC_KEY_LENGTH - keylen);
}
}
@@ -2655,9 +2640,8 @@ hifn_abort(struct hifn_softc *sc)
BUS_DMASYNC_POSTREAD);
}
- if (cmd->src_m != cmd->dst_m) {
- m_freem(cmd->src_m);
- crp->crp_buf = (caddr_t)cmd->dst_m;
+ if (cmd->dst_m != NULL) {
+ m_freem(cmd->dst_m);
}
/* non-shared buffers cannot be restarted */
@@ -2696,9 +2680,9 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
{
struct hifn_dma *dma = sc->sc_dma;
struct cryptop *crp = cmd->crp;
- struct cryptodesc *crd;
+ uint8_t macbuf2[SHA1_HASH_LEN];
struct mbuf *m;
- int totlen, i, u, ivlen;
+ int totlen, i, u;
if (cmd->src_map == cmd->dst_map) {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
@@ -2710,9 +2694,8 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
BUS_DMASYNC_POSTREAD);
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (cmd->src_m != cmd->dst_m) {
- crp->crp_buf = (caddr_t)cmd->dst_m;
+ if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ if (cmd->dst_m != NULL) {
totlen = cmd->src_mapsize;
for (m = cmd->dst_m; m != NULL; m = m->m_next) {
if (totlen < m->m_len) {
@@ -2721,15 +2704,15 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
} else
totlen -= m->m_len;
}
- cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
- m_freem(cmd->src_m);
+ cmd->dst_m->m_pkthdr.len = crp->crp_mbuf->m_pkthdr.len;
+ m_freem(crp->crp_mbuf);
+ crp->crp_mbuf = cmd->dst_m;
}
}
if (cmd->sloplen != 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
- (caddr_t)&dma->slop[cmd->slopidx]);
+ crypto_copyback(crp, cmd->src_mapsize - cmd->sloplen,
+ cmd->sloplen, &dma->slop[cmd->slopidx]);
}
i = sc->sc_dstk; u = sc->sc_dstu;
@@ -2749,37 +2732,16 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
hifnstats.hst_obytes += cmd->dst_mapsize;
- if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
- HIFN_BASE_CMD_CRYPT) {
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (crd->crd_alg != CRYPTO_DES_CBC &&
- crd->crd_alg != CRYPTO_3DES_CBC &&
- crd->crd_alg != CRYPTO_AES_CBC)
- continue;
- ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
- HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + crd->crd_len - ivlen, ivlen,
- cmd->session->hs_iv);
- break;
- }
- }
-
if (macbuf != NULL) {
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- int len;
-
- if (crd->crd_alg != CRYPTO_MD5 &&
- crd->crd_alg != CRYPTO_SHA1 &&
- crd->crd_alg != CRYPTO_MD5_HMAC &&
- crd->crd_alg != CRYPTO_SHA1_HMAC) {
- continue;
- }
- len = cmd->session->hs_mlen;
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, len, macbuf);
- break;
- }
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ cmd->session->hs_mlen, macbuf2);
+ if (timingsafe_bcmp(macbuf, macbuf2,
+ cmd->session->hs_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ cmd->session->hs_mlen, macbuf);
}
if (cmd->src_map != cmd->dst_map) {
diff --git a/sys/dev/hifn/hifn7751var.h b/sys/dev/hifn/hifn7751var.h
index e7ace8bcc977..ec0d96ec20a4 100644
--- a/sys/dev/hifn/hifn7751var.h
+++ b/sys/dev/hifn/hifn7751var.h
@@ -105,7 +105,6 @@ struct hifn_dma {
struct hifn_session {
- u_int8_t hs_iv[HIFN_MAX_IV_LENGTH];
int hs_mlen;
};
@@ -160,6 +159,7 @@ struct hifn_softc {
int sc_cmdk, sc_srck, sc_dstk, sc_resk;
int32_t sc_cid;
+ uint16_t sc_ena;
int sc_maxses;
int sc_ramsize;
int sc_flags;
@@ -257,10 +257,6 @@ struct hifn_softc {
*
*/
struct hifn_operand {
- union {
- struct mbuf *m;
- struct uio *io;
- } u;
bus_dmamap_t map;
bus_size_t mapsize;
int nsegs;
@@ -269,27 +265,24 @@ struct hifn_operand {
struct hifn_command {
struct hifn_session *session;
u_int16_t base_masks, cry_masks, mac_masks;
- u_int8_t iv[HIFN_MAX_IV_LENGTH], *ck, mac[HIFN_MAC_KEY_LENGTH];
+ u_int8_t iv[HIFN_MAX_IV_LENGTH], mac[HIFN_MAC_KEY_LENGTH];
+ const uint8_t *ck;
int cklen;
int sloplen, slopidx;
struct hifn_operand src;
struct hifn_operand dst;
+ struct mbuf *dst_m;
struct hifn_softc *softc;
struct cryptop *crp;
- struct cryptodesc *enccrd, *maccrd;
};
-#define src_m src.u.m
-#define src_io src.u.io
#define src_map src.map
#define src_mapsize src.mapsize
#define src_segs src.segs
#define src_nsegs src.nsegs
-#define dst_m dst.u.m
-#define dst_io dst.u.io
#define dst_map dst.map
#define dst_mapsize dst.mapsize
#define dst_segs dst.segs
diff --git a/sys/dev/safe/safe.c b/sys/dev/safe/safe.c
index 7a577dfd0a8c..99f16de56c50 100644
--- a/sys/dev/safe/safe.c
+++ b/sys/dev/safe/safe.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
+#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -56,10 +57,8 @@ __FBSDID("$FreeBSD$");
#include <sys/bus.h>
#include <sys/rman.h>
-#include <crypto/sha1.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
-#include <sys/md5.h>
+#include <opencrypto/xform_auth.h>
#include <sys/random.h>
#include <sys/kobj.h>
@@ -88,7 +87,9 @@ static int safe_suspend(device_t);
static int safe_resume(device_t);
static int safe_shutdown(device_t);
-static int safe_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int safe_probesession(device_t, const struct crypto_session_params *);
+static int safe_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int safe_process(device_t, struct cryptop *, int);
static device_method_t safe_methods[] = {
@@ -101,6 +102,7 @@ static device_method_t safe_methods[] = {
DEVMETHOD(device_shutdown, safe_shutdown),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, safe_probesession),
DEVMETHOD(cryptodev_newsession, safe_newsession),
DEVMETHOD(cryptodev_process, safe_process),
@@ -221,7 +223,7 @@ safe_attach(device_t dev)
{
struct safe_softc *sc = device_get_softc(dev);
u_int32_t raddr;
- u_int32_t i, devinfo;
+ u_int32_t i;
int rid;
bzero(sc, sizeof (*sc));
@@ -374,12 +376,12 @@ safe_attach(device_t dev)
device_printf(sc->sc_dev, "%s", safe_partname(sc));
- devinfo = READ_REG(sc, SAFE_DEVINFO);
- if (devinfo & SAFE_DEVINFO_RNG) {
+ sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO);
+ if (sc->sc_devinfo & SAFE_DEVINFO_RNG) {
sc->sc_flags |= SAFE_FLAGS_RNG;
printf(" rng");
}
- if (devinfo & SAFE_DEVINFO_PKEY) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) {
#if 0
printf(" key");
sc->sc_flags |= SAFE_FLAGS_KEY;
@@ -387,26 +389,18 @@ safe_attach(device_t dev)
crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
#endif
}
- if (devinfo & SAFE_DEVINFO_DES) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_DES) {
printf(" des/3des");
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
}
- if (devinfo & SAFE_DEVINFO_AES) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_AES) {
printf(" aes");
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
}
- if (devinfo & SAFE_DEVINFO_MD5) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_MD5) {
printf(" md5");
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
}
- if (devinfo & SAFE_DEVINFO_SHA1) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) {
printf(" sha1");
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
}
- printf(" null");
- crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
/* XXX other supported algorithms */
printf("\n");
@@ -629,11 +623,11 @@ safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
#define N(a) (sizeof(a) / sizeof (a[0]))
static void
-safe_setup_enckey(struct safe_session *ses, caddr_t key)
+safe_setup_enckey(struct safe_session *ses, const void *key)
{
int i;
- bcopy(key, ses->ses_key, ses->ses_klen / 8);
+ bcopy(key, ses->ses_key, ses->ses_klen);
/* PE is little-endian, insure proper byte order */
for (i = 0; i < N(ses->ses_key); i++)
@@ -641,47 +635,30 @@ safe_setup_enckey(struct safe_session *ses, caddr_t key)
}
static void
-safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
+safe_setup_mackey(struct safe_session *ses, int algo, const uint8_t *key,
+ int klen)
{
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
int i;
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
-
if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
- } else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_ipad_buffer,
- SHA1_BLOCK_LEN - klen);
- bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
- }
-
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
- if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
+
+ explicit_bzero(&md5ctx, sizeof(md5ctx));
} else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_opad_buffer,
- SHA1_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
+ bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
+
+ hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
- }
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ explicit_bzero(&sha1ctx, sizeof(sha1ctx));
+ }
/* PE is little-endian, insure proper byte order */
for (i = 0; i < N(ses->ses_hminner); i++) {
@@ -691,98 +668,147 @@ safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
}
#undef N
-/*
- * Allocate a new 'session' and return an encoded session id. 'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
- */
+static bool
+safe_auth_supported(struct safe_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_MD5) == 0)
+ return (false);
+ break;
+ case CRYPTO_SHA1_HMAC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+safe_cipher_supported(struct safe_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_DES) == 0)
+ return (false);
+ if (csp->csp_ivlen != 8)
+ return (false);
+ if (csp->csp_cipher_alg == CRYPTO_DES_CBC) {
+ if (csp->csp_cipher_klen != 8)
+ return (false);
+ } else {
+ if (csp->csp_cipher_klen != 24)
+ return (false);
+ }
+ break;
+ case CRYPTO_AES_CBC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0)
+ return (false);
+ if (csp->csp_ivlen != 16)
+ return (false);
+ if (csp->csp_cipher_klen != 16 &&
+ csp->csp_cipher_klen != 24 &&
+ csp->csp_cipher_klen != 32)
+ return (false);
+ break;
+ }
+ return (true);
+}
+
static int
-safe_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+safe_probesession(device_t dev, const struct crypto_session_params *csp)
{
struct safe_softc *sc = device_get_softc(dev);
- struct cryptoini *c, *encini = NULL, *macini = NULL;
- struct safe_session *ses = NULL;
- if (cri == NULL || sc == NULL)
+ if (csp->csp_flags != 0)
return (EINVAL);
-
- for (c = cri; c != NULL; c = c->cri_next) {
- if (c->cri_alg == CRYPTO_MD5_HMAC ||
- c->cri_alg == CRYPTO_SHA1_HMAC ||
- c->cri_alg == CRYPTO_NULL_HMAC) {
- if (macini)
- return (EINVAL);
- macini = c;
- } else if (c->cri_alg == CRYPTO_DES_CBC ||
- c->cri_alg == CRYPTO_3DES_CBC ||
- c->cri_alg == CRYPTO_AES_CBC ||
- c->cri_alg == CRYPTO_NULL_CBC) {
- if (encini)
- return (EINVAL);
- encini = c;
- } else
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!safe_auth_supported(sc, csp))
return (EINVAL);
- }
- if (encini == NULL && macini == NULL)
+ break;
+ case CSP_MODE_CIPHER:
+ if (!safe_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!safe_auth_supported(sc, csp) ||
+ !safe_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ default:
return (EINVAL);
- if (encini) { /* validate key length */
- switch (encini->cri_alg) {
- case CRYPTO_DES_CBC:
- if (encini->cri_klen != 64)
- return (EINVAL);
- break;
- case CRYPTO_3DES_CBC:
- if (encini->cri_klen != 192)
- return (EINVAL);
- break;
- case CRYPTO_AES_CBC:
- if (encini->cri_klen != 128 &&
- encini->cri_klen != 192 &&
- encini->cri_klen != 256)
- return (EINVAL);
- break;
- }
}
- ses = crypto_get_driver_session(cses);
- if (encini) {
- /* get an IV */
- /* XXX may read fewer than requested */
- read_random(ses->ses_iv, sizeof(ses->ses_iv));
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+/*
+ * Allocate a new 'session'.
+ */
+static int
+safe_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct safe_session *ses;
- ses->ses_klen = encini->cri_klen;
- if (encini->cri_key != NULL)
- safe_setup_enckey(ses, encini->cri_key);
+ ses = crypto_get_driver_session(cses);
+ if (csp->csp_cipher_alg != 0) {
+ ses->ses_klen = csp->csp_cipher_klen;
+ if (csp->csp_cipher_key != NULL)
+ safe_setup_enckey(ses, csp->csp_cipher_key);
}
- if (macini) {
- ses->ses_mlen = macini->cri_mlen;
+ if (csp->csp_auth_alg != 0) {
+ ses->ses_mlen = csp->csp_auth_mlen;
if (ses->ses_mlen == 0) {
- if (macini->cri_alg == CRYPTO_MD5_HMAC)
+ if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
ses->ses_mlen = MD5_HASH_LEN;
else
ses->ses_mlen = SHA1_HASH_LEN;
}
- if (macini->cri_key != NULL) {
- safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
- macini->cri_klen / 8);
+ if (csp->csp_auth_key != NULL) {
+ safe_setup_mackey(ses, csp->csp_auth_alg,
+ csp->csp_auth_key, csp->csp_auth_klen);
}
}
return (0);
}
+static bus_size_t
+safe_crp_length(struct cryptop *crp)
+{
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (crp->crp_mbuf->m_pkthdr.len);
+ case CRYPTO_BUF_UIO:
+ return (crp->crp_uio->uio_resid);
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_ilen);
+ default:
+ panic("bad crp buffer type");
+ }
+}
+
static void
-safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
{
struct safe_operand *op = arg;
- DPRINTF(("%s: mapsize %u nsegs %d error %d\n", __func__,
- (u_int) mapsize, nsegs, error));
+ DPRINTF(("%s: nsegs %d error %d\n", __func__,
+ nsegs, error));
if (error != 0)
return;
- op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
@@ -790,11 +816,10 @@ safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int
static int
safe_process(device_t dev, struct cryptop *crp, int hint)
{
- struct safe_softc *sc = device_get_softc(dev);
+ struct safe_softc *sc = device_get_softc(dev);
+ const struct crypto_session_params *csp;
int err = 0, i, nicealign, uniform;
- struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
- int bypass, oplen, ivsize;
- caddr_t iv;
+ int bypass, oplen;
int16_t coffset;
struct safe_session *ses;
struct safe_ringentry *re;
@@ -802,11 +827,6 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
struct safe_pdesc *pd;
u_int32_t cmd0, cmd1, staterec;
- if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
- safestats.st_invalid++;
- return (EINVAL);
- }
-
mtx_lock(&sc->sc_ringmtx);
if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
safestats.st_ringfull++;
@@ -823,104 +843,46 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
re->re_crp = crp;
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- re->re_src_m = (struct mbuf *)crp->crp_buf;
- re->re_dst_m = (struct mbuf *)crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- re->re_src_io = (struct uio *)crp->crp_buf;
- re->re_dst_io = (struct uio *)crp->crp_buf;
- } else {
- safestats.st_badflags++;
- err = EINVAL;
- goto errout; /* XXX we don't handle contiguous blocks! */
- }
-
sa = &re->re_sa;
ses = crypto_get_driver_session(crp->crp_session);
-
- crd1 = crp->crp_desc;
- if (crd1 == NULL) {
- safestats.st_nodesc++;
- err = EINVAL;
- goto errout;
- }
- crd2 = crd1->crd_next;
+ csp = crypto_get_params(crp->crp_session);
cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
cmd1 = 0;
- if (crd2 == NULL) {
- if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_NULL_HMAC) {
- maccrd = crd1;
- enccrd = NULL;
- cmd0 |= SAFE_SA_CMD0_OP_HASH;
- } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_NULL_CBC) {
- maccrd = NULL;
- enccrd = crd1;
- cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
- } else {
- safestats.st_badalg++;
- err = EINVAL;
- goto errout;
- }
- } else {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_NULL_HMAC) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC ||
- crd2->crd_alg == CRYPTO_AES_CBC ||
- crd2->crd_alg == CRYPTO_NULL_CBC) &&
- ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
- maccrd = crd1;
- enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_NULL_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC ||
- crd2->crd_alg == CRYPTO_NULL_HMAC) &&
- (crd1->crd_flags & CRD_F_ENCRYPT)) {
- enccrd = crd1;
- maccrd = crd2;
- } else {
- safestats.st_badalg++;
- err = EINVAL;
- goto errout;
- }
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ cmd0 |= SAFE_SA_CMD0_OP_HASH;
+ break;
+ case CSP_MODE_CIPHER:
+ cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
+ break;
+ case CSP_MODE_ETA:
cmd0 |= SAFE_SA_CMD0_OP_BOTH;
+ break;
}
- if (enccrd) {
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
- safe_setup_enckey(ses, enccrd->crd_key);
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_cipher_key != NULL)
+ safe_setup_enckey(ses, crp->crp_cipher_key);
- if (enccrd->crd_alg == CRYPTO_DES_CBC) {
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
cmd0 |= SAFE_SA_CMD0_DES;
cmd1 |= SAFE_SA_CMD1_CBC;
- ivsize = 2*sizeof(u_int32_t);
- } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
+ break;
+ case CRYPTO_3DES_CBC:
cmd0 |= SAFE_SA_CMD0_3DES;
cmd1 |= SAFE_SA_CMD1_CBC;
- ivsize = 2*sizeof(u_int32_t);
- } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
+ break;
+ case CRYPTO_AES_CBC:
cmd0 |= SAFE_SA_CMD0_AES;
cmd1 |= SAFE_SA_CMD1_CBC;
- if (ses->ses_klen == 128)
+ if (ses->ses_klen * 8 == 128)
cmd1 |= SAFE_SA_CMD1_AES128;
- else if (ses->ses_klen == 192)
+ else if (ses->ses_klen * 8 == 192)
cmd1 |= SAFE_SA_CMD1_AES192;
else
cmd1 |= SAFE_SA_CMD1_AES256;
- ivsize = 4*sizeof(u_int32_t);
- } else {
- cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
- ivsize = 0;
}
/*
@@ -932,32 +894,28 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
* in the state record and set the hash/crypt offset to
* copy both the header+IV.
*/
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(re->re_sastate.sa_saved_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+ re->re_sastate.sa_saved_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(re->re_sastate.sa_saved_iv, crp->crp_iv,
+ csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ re->re_sastate.sa_saved_iv);
+ cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
+
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
cmd0 |= SAFE_SA_CMD0_OUTBOUND;
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- iv = enccrd->crd_iv;
- else
- iv = (caddr_t) ses->ses_iv;
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivsize, iv);
- }
- bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
- cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
- re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
+ /*
+ * XXX: I suspect we don't need this since we
+ * don't save the returned IV.
+ */
+ cmd0 |= SAFE_SA_CMD0_SAVEIV;
} else {
cmd0 |= SAFE_SA_CMD0_INBOUND;
-
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
- bcopy(enccrd->crd_iv,
- re->re_sastate.sa_saved_iv, ivsize);
- } else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivsize,
- (caddr_t)re->re_sastate.sa_saved_iv);
- }
- cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
}
/*
* For basic encryption use the zero pad algorithm.
@@ -973,21 +931,23 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
}
- if (maccrd) {
- if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- safe_setup_mackey(ses, maccrd->crd_alg,
- maccrd->crd_key, maccrd->crd_klen / 8);
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_auth_key != NULL) {
+ safe_setup_mackey(ses, csp->csp_auth_alg,
+ crp->crp_auth_key, csp->csp_auth_klen);
}
- if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
cmd0 |= SAFE_SA_CMD0_MD5;
cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
- } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
+ break;
+ case CRYPTO_SHA1_HMAC:
cmd0 |= SAFE_SA_CMD0_SHA1;
cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
- } else {
- cmd0 |= SAFE_SA_CMD0_HASH_NULL;
+ break;
}
+
/*
* Digest data is loaded from the SA and the hash
* result is saved to the state block where we
@@ -1003,38 +963,32 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
}
- if (enccrd && maccrd) {
+ if (csp->csp_mode == CSP_MODE_ETA) {
/*
- * The offset from hash data to the start of
- * crypt data is the difference in the skips.
+ * The driver only supports ETA requests where there
+ * is no gap between the AAD and payload.
*/
- bypass = maccrd->crd_skip;
- coffset = enccrd->crd_skip - maccrd->crd_skip;
- if (coffset < 0) {
- DPRINTF(("%s: hash does not precede crypt; "
- "mac skip %u enc skip %u\n",
- __func__, maccrd->crd_skip, enccrd->crd_skip));
- safestats.st_skipmismatch++;
- err = EINVAL;
- goto errout;
- }
- oplen = enccrd->crd_skip + enccrd->crd_len;
- if (maccrd->crd_skip + maccrd->crd_len != oplen) {
- DPRINTF(("%s: hash amount %u != crypt amount %u\n",
- __func__, maccrd->crd_skip + maccrd->crd_len,
- oplen));
+ if (crp->crp_aad_length != 0 &&
+ crp->crp_aad_start + crp->crp_aad_length !=
+ crp->crp_payload_start) {
safestats.st_lenmismatch++;
err = EINVAL;
goto errout;
}
+ if (crp->crp_aad_length != 0)
+ bypass = crp->crp_aad_start;
+ else
+ bypass = crp->crp_payload_start;
+ coffset = crp->crp_aad_length;
+ oplen = crp->crp_payload_start + crp->crp_payload_length;
#ifdef SAFE_DEBUG
if (safe_debug) {
- printf("mac: skip %d, len %d, inject %d\n",
- maccrd->crd_skip, maccrd->crd_len,
- maccrd->crd_inject);
- printf("enc: skip %d, len %d, inject %d\n",
- enccrd->crd_skip, enccrd->crd_len,
- enccrd->crd_inject);
+ printf("AAD: skip %d, len %d, digest %d\n",
+ crp->crp_aad_start, crp->crp_aad_length,
+ crp->crp_digest_start);
+ printf("payload: skip %d, len %d, IV %d\n",
+ crp->crp_payload_start, crp->crp_payload_length,
+ crp->crp_iv_start);
printf("bypass %d coffset %d oplen %d\n",
bypass, coffset, oplen);
}
@@ -1070,13 +1024,8 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
*/
cmd1 |= SAFE_SA_CMD1_MUTABLE;
} else {
- if (enccrd) {
- bypass = enccrd->crd_skip;
- oplen = bypass + enccrd->crd_len;
- } else {
- bypass = maccrd->crd_skip;
- oplen = bypass + maccrd->crd_len;
- }
+ bypass = crp->crp_payload_start;
+ oplen = bypass + crp->crp_payload_length;
coffset = 0;
}
/* XXX verify multiple of 4 when using s/g */
@@ -1092,27 +1041,15 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
err = ENOMEM;
goto errout;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_srcdmat, re->re_src_map,
- re->re_src_m, safe_op_cb,
- &re->re_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
- re->re_src_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_srcdmat, re->re_src_map,
- re->re_src_io, safe_op_cb,
- &re->re_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
- re->re_src_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
+ if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,
+ &re->re_src, BUS_DMA_NOWAIT) != 0) {
+ bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
+ re->re_src_map = NULL;
+ safestats.st_noload++;
+ err = ENOMEM;
+ goto errout;
}
+ re->re_src_mapsize = safe_crp_length(crp);
nicealign = safe_dmamap_aligned(&re->re_src);
uniform = safe_dmamap_uniform(&re->re_src);
@@ -1143,211 +1080,175 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
re->re_desc.d_src = re->re_src_segs[0].ds_addr;
}
- if (enccrd == NULL && maccrd != NULL) {
+ if (csp->csp_mode == CSP_MODE_DIGEST) {
/*
* Hash op; no destination needed.
*/
} else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- if (!nicealign) {
- safestats.st_iovmisaligned++;
- err = EINVAL;
+ if (nicealign && uniform == 1) {
+ /*
+ * Source layout is suitable for direct
+ * sharing of the DMA map and segment list.
+ */
+ re->re_dst = re->re_src;
+ } else if (nicealign && uniform == 2) {
+ /*
+ * The source is properly aligned but requires a
+ * different particle list to handle DMA of the
+ * result. Create a new map and do the load to
+ * create the segment list. The particle
+ * descriptor setup code below will handle the
+ * rest.
+ */
+ if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT,
+ &re->re_dst_map)) {
+ safestats.st_nomap++;
+ err = ENOMEM;
goto errout;
}
- if (uniform != 1) {
- /*
- * Source is not suitable for direct use as
- * the destination. Create a new scatter/gather
- * list based on the destination requirements
- * and check if that's ok.
- */
- if (bus_dmamap_create(sc->sc_dstdmat,
- BUS_DMA_NOWAIT, &re->re_dst_map)) {
- safestats.st_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_uio(sc->sc_dstdmat,
- re->re_dst_map, re->re_dst_io,
- safe_op_cb, &re->re_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dstdmat,
- re->re_dst_map);
- re->re_dst_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
- uniform = safe_dmamap_uniform(&re->re_dst);
- if (!uniform) {
- /*
- * There's no way to handle the DMA
- * requirements with this uio. We
- * could create a separate DMA area for
- * the result and then copy it back,
- * but for now we just bail and return
- * an error. Note that uio requests
- * > SAFE_MAX_DSIZE are handled because
- * the DMA map and segment list for the
- * destination wil result in a
- * destination particle list that does
- * the necessary scatter DMA.
- */
- safestats.st_iovnotuniform++;
- err = EINVAL;
- goto errout;
- }
- } else
- re->re_dst = re->re_src;
- } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (nicealign && uniform == 1) {
- /*
- * Source layout is suitable for direct
- * sharing of the DMA map and segment list.
- */
- re->re_dst = re->re_src;
- } else if (nicealign && uniform == 2) {
- /*
- * The source is properly aligned but requires a
- * different particle list to handle DMA of the
- * result. Create a new map and do the load to
- * create the segment list. The particle
- * descriptor setup code below will handle the
- * rest.
- */
- if (bus_dmamap_create(sc->sc_dstdmat,
- BUS_DMA_NOWAIT, &re->re_dst_map)) {
- safestats.st_nomap++;
- err = ENOMEM;
- goto errout;
+ if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,
+ crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=
+ 0) {
+ bus_dmamap_destroy(sc->sc_dstdmat,
+ re->re_dst_map);
+ re->re_dst_map = NULL;
+ safestats.st_noload++;
+ err = ENOMEM;
+ goto errout;
+ }
+ } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ int totlen, len;
+ struct mbuf *m, *top, **mp;
+
+ /*
+ * DMA constraints require that we allocate a
+ * new mbuf chain for the destination. We
+ * allocate an entire new set of mbufs of
+ * optimal/required size and then tell the
+ * hardware to copy any bits that are not
+ * created as a byproduct of the operation.
+ */
+ if (!nicealign)
+ safestats.st_unaligned++;
+ if (!uniform)
+ safestats.st_notuniform++;
+ totlen = re->re_src_mapsize;
+ if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+ len = MHLEN;
+ MGETHDR(m, M_NOWAIT, MT_DATA);
+ if (m && !m_dup_pkthdr(m, crp->crp_mbuf,
+ M_NOWAIT)) {
+ m_free(m);
+ m = NULL;
}
- if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
- re->re_dst_map, re->re_dst_m,
- safe_op_cb, &re->re_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dstdmat,
- re->re_dst_map);
- re->re_dst_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
+ } else {
+ len = MLEN;
+ MGET(m, M_NOWAIT, MT_DATA);
+ }
+ if (m == NULL) {
+ safestats.st_nombuf++;
+ err = sc->sc_nqchip ? ERESTART : ENOMEM;
+ goto errout;
+ }
+ if (totlen >= MINCLSIZE) {
+ if (!(MCLGET(m, M_NOWAIT))) {
+ m_free(m);
+ safestats.st_nomcl++;
+ err = sc->sc_nqchip ?
+ ERESTART : ENOMEM;
goto errout;
}
- } else { /* !(aligned and/or uniform) */
- int totlen, len;
- struct mbuf *m, *top, **mp;
+ len = MCLBYTES;
+ }
+ m->m_len = len;
+ top = NULL;
+ mp = &top;
- /*
- * DMA constraints require that we allocate a
- * new mbuf chain for the destination. We
- * allocate an entire new set of mbufs of
- * optimal/required size and then tell the
- * hardware to copy any bits that are not
- * created as a byproduct of the operation.
- */
- if (!nicealign)
- safestats.st_unaligned++;
- if (!uniform)
- safestats.st_notuniform++;
- totlen = re->re_src_mapsize;
- if (re->re_src_m->m_flags & M_PKTHDR) {
- len = MHLEN;
- MGETHDR(m, M_NOWAIT, MT_DATA);
- if (m && !m_dup_pkthdr(m, re->re_src_m,
- M_NOWAIT)) {
- m_free(m);
- m = NULL;
+ while (totlen > 0) {
+ if (top) {
+ MGET(m, M_NOWAIT, MT_DATA);
+ if (m == NULL) {
+ m_freem(top);
+ safestats.st_nombuf++;
+ err = sc->sc_nqchip ?
+ ERESTART : ENOMEM;
+ goto errout;
}
- } else {
len = MLEN;
- MGET(m, M_NOWAIT, MT_DATA);
- }
- if (m == NULL) {
- safestats.st_nombuf++;
- err = sc->sc_nqchip ? ERESTART : ENOMEM;
- goto errout;
}
- if (totlen >= MINCLSIZE) {
+ if (top && totlen >= MINCLSIZE) {
if (!(MCLGET(m, M_NOWAIT))) {
- m_free(m);
+ *mp = m;
+ m_freem(top);
safestats.st_nomcl++;
err = sc->sc_nqchip ?
- ERESTART : ENOMEM;
+ ERESTART : ENOMEM;
goto errout;
}
len = MCLBYTES;
}
- m->m_len = len;
- top = NULL;
- mp = &top;
-
- while (totlen > 0) {
- if (top) {
- MGET(m, M_NOWAIT, MT_DATA);
- if (m == NULL) {
- m_freem(top);
- safestats.st_nombuf++;
- err = sc->sc_nqchip ?
- ERESTART : ENOMEM;
- goto errout;
- }
- len = MLEN;
- }
- if (top && totlen >= MINCLSIZE) {
- if (!(MCLGET(m, M_NOWAIT))) {
- *mp = m;
- m_freem(top);
- safestats.st_nomcl++;
- err = sc->sc_nqchip ?
- ERESTART : ENOMEM;
- goto errout;
- }
- len = MCLBYTES;
- }
- m->m_len = len = min(totlen, len);
- totlen -= len;
- *mp = m;
- mp = &m->m_next;
- }
- re->re_dst_m = top;
- if (bus_dmamap_create(sc->sc_dstdmat,
- BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
- safestats.st_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
- re->re_dst_map, re->re_dst_m,
- safe_op_cb, &re->re_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dstdmat,
- re->re_dst_map);
- re->re_dst_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
- if (re->re_src.mapsize > oplen) {
- /*
- * There's data following what the
- * hardware will copy for us. If this
- * isn't just the ICV (that's going to
- * be written on completion), copy it
- * to the new mbufs
- */
- if (!(maccrd &&
- (re->re_src.mapsize-oplen) == 12 &&
- maccrd->crd_inject == oplen))
- safe_mcopy(re->re_src_m,
- re->re_dst_m,
- oplen);
- else
- safestats.st_noicvcopy++;
- }
+ m->m_len = len = min(totlen, len);
+ totlen -= len;
+ *mp = m;
+ mp = &m->m_next;
+ }
+ re->re_dst_m = top;
+ if (bus_dmamap_create(sc->sc_dstdmat,
+ BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
+ safestats.st_nomap++;
+ err = ENOMEM;
+ goto errout;
+ }
+ if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat,
+ re->re_dst_map, top, re->re_dst_segs,
+ &re->re_dst_nsegs, 0) != 0) {
+ bus_dmamap_destroy(sc->sc_dstdmat,
+ re->re_dst_map);
+ re->re_dst_map = NULL;
+ safestats.st_noload++;
+ err = ENOMEM;
+ goto errout;
+ }
+ re->re_dst_mapsize = re->re_src_mapsize;
+ if (re->re_src.mapsize > oplen) {
+ /*
+ * There's data following what the
+ * hardware will copy for us. If this
+ * isn't just the ICV (that's going to
+ * be written on completion), copy it
+ * to the new mbufs
+ */
+ if (!(csp->csp_mode == CSP_MODE_ETA &&
+ (re->re_src.mapsize-oplen) == ses->ses_mlen &&
+ crp->crp_digest_start == oplen))
+ safe_mcopy(crp->crp_mbuf, re->re_dst_m,
+ oplen);
+ else
+ safestats.st_noicvcopy++;
}
} else {
- safestats.st_badflags++;
- err = EINVAL;
- goto errout;
+ if (!nicealign) {
+ safestats.st_iovmisaligned++;
+ err = EINVAL;
+ goto errout;
+ } else {
+ /*
+ * There's no way to handle the DMA
+ * requirements with this uio. We
+ * could create a separate DMA area for
+ * the result and then copy it back,
+ * but for now we just bail and return
+ * an error. Note that uio requests
+ * > SAFE_MAX_DSIZE are handled because
+ * the DMA map and segment list for the
+ * destination wil result in a
+ * destination particle list that does
+ * the necessary scatter DMA.
+ */
+ safestats.st_iovnotuniform++;
+ err = EINVAL;
+ goto errout;
+ }
}
if (re->re_dst.nsegs > 1) {
@@ -1393,7 +1294,7 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
* ready for processing.
*/
re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
- if (maccrd)
+ if (csp->csp_auth_alg != 0)
re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
re->re_desc.d_len = oplen
| SAFE_PE_LEN_READY
@@ -1412,7 +1313,7 @@ safe_process(device_t dev, struct cryptop *crp, int hint)
return (0);
errout:
- if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
+ if (re->re_dst_m != NULL)
m_freem(re->re_dst_m);
if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
@@ -1436,11 +1337,13 @@ errout:
static void
safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
{
+ const struct crypto_session_params *csp;
struct cryptop *crp = (struct cryptop *)re->re_crp;
struct safe_session *ses;
- struct cryptodesc *crd;
+ uint8_t hash[HASH_MAX_LEN];
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
safestats.st_opackets++;
safestats.st_obytes += re->re_dst.mapsize;
@@ -1454,6 +1357,9 @@ safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
safestats.st_peoperr++;
crp->crp_etype = EIO; /* something more meaningful? */
}
+
+ /* XXX: Should crp_mbuf be updated to re->re_dst_m if it is non-NULL? */
+
if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
BUS_DMASYNC_POSTREAD);
@@ -1464,58 +1370,29 @@ safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
- /*
- * If result was written to a differet mbuf chain, swap
- * it in as the return value and reclaim the original.
- */
- if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
- m_freem(re->re_src_m);
- crp->crp_buf = (caddr_t)re->re_dst_m;
- }
-
- if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
- /* copy out IV for future use */
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- int ivsize;
-
- if (crd->crd_alg == CRYPTO_DES_CBC ||
- crd->crd_alg == CRYPTO_3DES_CBC) {
- ivsize = 2*sizeof(u_int32_t);
- } else if (crd->crd_alg == CRYPTO_AES_CBC) {
- ivsize = 4*sizeof(u_int32_t);
- } else
- continue;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + crd->crd_len - ivsize, ivsize,
- (caddr_t)ses->ses_iv);
- break;
- }
- }
-
if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
- /* copy out ICV result */
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
- crd->crd_alg == CRYPTO_SHA1_HMAC ||
- crd->crd_alg == CRYPTO_NULL_HMAC))
- continue;
- if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
- /*
- * SHA-1 ICV's are byte-swapped; fix 'em up
- * before copy them to their destination.
- */
- re->re_sastate.sa_saved_indigest[0] =
- bswap32(re->re_sastate.sa_saved_indigest[0]);
- re->re_sastate.sa_saved_indigest[1] =
- bswap32(re->re_sastate.sa_saved_indigest[1]);
- re->re_sastate.sa_saved_indigest[2] =
- bswap32(re->re_sastate.sa_saved_indigest[2]);
- }
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, ses->ses_mlen,
- (caddr_t)re->re_sastate.sa_saved_indigest);
- break;
+ if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) {
+ /*
+ * SHA-1 ICV's are byte-swapped; fix 'em up
+ * before copying them to their destination.
+ */
+ re->re_sastate.sa_saved_indigest[0] =
+ bswap32(re->re_sastate.sa_saved_indigest[0]);
+ re->re_sastate.sa_saved_indigest[1] =
+ bswap32(re->re_sastate.sa_saved_indigest[1]);
+ re->re_sastate.sa_saved_indigest[2] =
+ bswap32(re->re_sastate.sa_saved_indigest[2]);
}
+
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ ses->ses_mlen, hash);
+ if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,
+ hash, ses->ses_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ ses->ses_mlen, re->re_sastate.sa_saved_indigest);
}
crypto_done(crp);
}
@@ -1921,7 +1798,7 @@ safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
/*
* Free header MCR
*/
- if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
+ if (re->re_dst_m != NULL)
m_freem(re->re_dst_m);
crp = (struct cryptop *)re->re_crp;
diff --git a/sys/dev/safe/safevar.h b/sys/dev/safe/safevar.h
index 024d00564562..e7d238669ed6 100644
--- a/sys/dev/safe/safevar.h
+++ b/sys/dev/safe/safevar.h
@@ -75,10 +75,6 @@ struct safe_dma_alloc {
* where each is mapped for DMA.
*/
struct safe_operand {
- union {
- struct mbuf *m;
- struct uio *io;
- } u;
bus_dmamap_t map;
bus_size_t mapsize;
int nsegs;
@@ -109,22 +105,18 @@ struct safe_ringentry {
struct safe_operand re_src; /* source operand */
struct safe_operand re_dst; /* destination operand */
+ struct mbuf *re_dst_m;
int unused;
int re_flags;
-#define SAFE_QFLAGS_COPYOUTIV 0x1 /* copy back on completion */
#define SAFE_QFLAGS_COPYOUTICV 0x2 /* copy back on completion */
};
-#define re_src_m re_src.u.m
-#define re_src_io re_src.u.io
#define re_src_map re_src.map
#define re_src_nsegs re_src.nsegs
#define re_src_segs re_src.segs
#define re_src_mapsize re_src.mapsize
-#define re_dst_m re_dst.u.m
-#define re_dst_io re_dst.u.io
#define re_dst_map re_dst.map
#define re_dst_nsegs re_dst.nsegs
#define re_dst_segs re_dst.segs
@@ -138,7 +130,6 @@ struct safe_session {
u_int32_t ses_mlen; /* hmac length in bytes */
u_int32_t ses_hminner[5]; /* hmac inner state */
u_int32_t ses_hmouter[5]; /* hmac outer state */
- u_int32_t ses_iv[4]; /* DES/3DES/AES iv */
};
struct safe_softc {
@@ -157,6 +148,7 @@ struct safe_softc {
int sc_suspended;
int sc_needwakeup; /* notify crypto layer */
int32_t sc_cid; /* crypto tag */
+ uint32_t sc_devinfo;
struct safe_dma_alloc sc_ringalloc; /* PE ring allocation state */
struct safe_ringentry *sc_ring; /* PE ring */
struct safe_ringentry *sc_ringtop; /* PE ring top */
diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c
index 76f808757845..3b3ea0018060 100644
--- a/sys/dev/sec/sec.c
+++ b/sys/dev/sec/sec.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
#include <machine/resource.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
#include "cryptodev_if.h"
#include <dev/ofw/ofw_bus_subr.h>
@@ -74,7 +75,7 @@ static int sec_init(struct sec_softc *sc);
static int sec_alloc_dma_mem(struct sec_softc *sc,
struct sec_dma_mem *dma_mem, bus_size_t size);
static int sec_desc_map_dma(struct sec_softc *sc,
- struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type,
+ struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size,
struct sec_desc_map_info *sdmi);
static void sec_free_dma_mem(struct sec_dma_mem *dma_mem);
static void sec_enqueue(struct sec_softc *sc);
@@ -82,48 +83,43 @@ static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
int channel);
static int sec_eu_channel(struct sec_softc *sc, int eu);
static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
- u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype);
+ u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize);
static int sec_make_pointer_direct(struct sec_softc *sc,
struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
+static int sec_probesession(device_t dev,
+ const struct crypto_session_params *csp);
static int sec_newsession(device_t dev, crypto_session_t cses,
- struct cryptoini *cri);
+ const struct crypto_session_params *csp);
static int sec_process(device_t dev, struct cryptop *crp, int hint);
-static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
- struct cryptoini **mac);
-static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
- struct cryptodesc **mac);
static int sec_build_common_ns_desc(struct sec_softc *sc,
- struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
- struct cryptodesc *enc, int buftype);
+ struct sec_desc *desc, const struct crypto_session_params *csp,
+ struct cryptop *crp);
static int sec_build_common_s_desc(struct sec_softc *sc,
- struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
- struct cryptodesc *enc, struct cryptodesc *mac, int buftype);
+ struct sec_desc *desc, const struct crypto_session_params *csp,
+ struct cryptop *crp);
static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
/* AESU */
-static int sec_aesu_newsession(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool sec_aesu_newsession(const struct crypto_session_params *csp);
static int sec_aesu_make_desc(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
- int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
/* DEU */
-static int sec_deu_newsession(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool sec_deu_newsession(const struct crypto_session_params *csp);
static int sec_deu_make_desc(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
- int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
/* MDEU */
-static int sec_mdeu_can_handle(u_int alg);
-static int sec_mdeu_config(struct cryptodesc *crd,
+static bool sec_mdeu_can_handle(u_int alg);
+static int sec_mdeu_config(const struct crypto_session_params *csp,
u_int *eu, u_int *mode, u_int *hashlen);
-static int sec_mdeu_newsession(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool sec_mdeu_newsession(const struct crypto_session_params *csp);
static int sec_mdeu_make_desc(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
- int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
static device_method_t sec_methods[] = {
/* Device interface */
@@ -136,6 +132,7 @@ static device_method_t sec_methods[] = {
DEVMETHOD(device_shutdown, sec_shutdown),
/* Crypto methods */
+ DEVMETHOD(cryptodev_probesession, sec_probesession),
DEVMETHOD(cryptodev_newsession, sec_newsession),
DEVMETHOD(cryptodev_process, sec_process),
@@ -362,24 +359,6 @@ sec_attach(device_t dev)
if (error)
goto fail6;
- /* Register in OCF (AESU) */
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-
- /* Register in OCF (DEU) */
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-
- /* Register in OCF (MDEU) */
- crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- if (sc->sc_version >= 3) {
- crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
- }
-
return (0);
fail6:
@@ -545,9 +524,12 @@ sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
static void
sec_primary_intr(void *arg)
{
+ struct sec_session *ses;
struct sec_softc *sc = arg;
struct sec_desc *desc;
+ struct cryptop *crp;
uint64_t isr;
+ uint8_t hash[HASH_MAX_LEN];
int i, wakeup = 0;
SEC_LOCK(sc, controller);
@@ -595,7 +577,26 @@ sec_primary_intr(void *arg)
SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
BUS_DMASYNC_PREWRITE);
- desc->sd_crp->crp_etype = desc->sd_error;
+ crp = desc->sd_crp;
+ crp->crp_etype = desc->sd_error;
+ if (crp->crp_etype == 0) {
+ ses = crypto_get_driver_session(crp->crp_session);
+ if (ses->ss_mlen != 0) {
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp,
+ crp->crp_digest_start,
+ ses->ss_mlen, hash);
+ if (timingsafe_bcmp(
+ desc->sd_desc->shd_digest,
+ hash, ses->ss_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp,
+ crp->crp_digest_start,
+ ses->ss_mlen,
+ desc->sd_desc->shd_digest);
+ }
+ }
crypto_done(desc->sd_crp);
SEC_DESC_FREE_POINTERS(desc);
@@ -786,14 +787,6 @@ sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
sdmi->sdmi_lt_last = lt;
}
-static void
-sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
- bus_size_t size, int error)
-{
-
- sec_dma_map_desc_cb(arg, segs, nseg, error);
-}
-
static int
sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
bus_size_t size)
@@ -851,22 +844,22 @@ err1:
}
static int
-sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
- bus_size_t size, int type, struct sec_desc_map_info *sdmi)
+sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
+ struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi)
{
int error;
if (dma_mem->dma_vaddr != NULL)
return (EBUSY);
- switch (type) {
- case SEC_MEMORY:
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_CONTIG:
break;
- case SEC_UIO:
+ case CRYPTO_BUF_UIO:
size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
break;
- case SEC_MBUF:
- size = m_length((struct mbuf*)mem, NULL);
+ case CRYPTO_BUF_MBUF:
+ size = m_length(crp->crp_mbuf, NULL);
break;
default:
return (EINVAL);
@@ -899,20 +892,8 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
return (error);
}
- switch (type) {
- case SEC_MEMORY:
- error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
- mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
- break;
- case SEC_UIO:
- error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map,
- mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
- break;
- case SEC_MBUF:
- error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map,
- mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
- break;
- }
+ error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp,
+ sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
if (error) {
device_printf(sc->sc_dev, "cannot get address of the DMA"
@@ -923,7 +904,7 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
}
dma_mem->dma_is_map = 1;
- dma_mem->dma_vaddr = mem;
+ dma_mem->dma_vaddr = crp;
return (0);
}
@@ -1130,7 +1111,7 @@ sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
static int
sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
- u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype)
+ u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize)
{
struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
struct sec_hw_desc_ptr *ptr;
@@ -1138,14 +1119,8 @@ sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
SEC_LOCK_ASSERT(sc, descriptors);
- /* For flat memory map only requested region */
- if (dtype == SEC_MEMORY) {
- data = (uint8_t*)(data) + doffset;
- sdmi.sdmi_offset = 0;
- }
-
- error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize,
- dtype, &sdmi);
+ error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize,
+ &sdmi);
if (error)
return (error);
@@ -1162,115 +1137,116 @@ sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
return (0);
}
-static int
-sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
- struct cryptoini **mac)
+static bool
+sec_cipher_supported(const struct crypto_session_params *csp)
{
- struct cryptoini *e, *m;
-
- e = cri;
- m = cri->cri_next;
- /* We can haldle only two operations */
- if (m && m->cri_next)
- return (EINVAL);
-
- if (sec_mdeu_can_handle(e->cri_alg)) {
- cri = m;
- m = e;
- e = cri;
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ /* AESU */
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ /* DEU */
+ if (csp->csp_ivlen != DES_BLOCK_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
}
- if (m && !sec_mdeu_can_handle(m->cri_alg))
- return (EINVAL);
+ if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN)
+ return (false);
- *enc = e;
- *mac = m;
+ return (true);
+}
- return (0);
+static bool
+sec_auth_supported(struct sec_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ if (sc->sc_version < 3)
+ return (false);
+ /* FALLTHROUGH */
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ if (csp->csp_auth_klen > SEC_MAX_KEY_LEN)
+ return (false);
+ break;
+ case CRYPTO_MD5:
+ case CRYPTO_SHA1:
+ break;
+ default:
+ return (false);
+ }
+ return (true);
}
static int
-sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
- struct cryptodesc **mac)
+sec_probesession(device_t dev, const struct crypto_session_params *csp)
{
- struct cryptodesc *e, *m, *t;
-
- e = crp->crp_desc;
- m = e->crd_next;
+ struct sec_softc *sc = device_get_softc(dev);
- /* We can haldle only two operations */
- if (m && m->crd_next)
+ if (csp->csp_flags != 0)
return (EINVAL);
-
- if (sec_mdeu_can_handle(e->crd_alg)) {
- t = m;
- m = e;
- e = t;
- }
-
- if (m && !sec_mdeu_can_handle(m->crd_alg))
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!sec_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!sec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
return (EINVAL);
-
- *enc = e;
- *mac = m;
-
- return (0);
+ }
+ return (CRYPTODEV_PROBE_HARDWARE);
}
static int
-sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+sec_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
- struct sec_softc *sc = device_get_softc(dev);
struct sec_eu_methods *eu = sec_eus;
- struct cryptoini *enc = NULL;
- struct cryptoini *mac = NULL;
struct sec_session *ses;
- int error = -1;
-
- error = sec_split_cri(cri, &enc, &mac);
- if (error)
- return (error);
-
- /* Check key lengths */
- if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN)
- return (E2BIG);
-
- if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN)
- return (E2BIG);
-
- /* Only SEC 3.0 supports digests larger than 256 bits */
- if (sc->sc_version < 3 && mac && mac->cri_klen > 256)
- return (E2BIG);
ses = crypto_get_driver_session(cses);
/* Find EU for this session */
while (eu->sem_make_desc != NULL) {
- error = eu->sem_newsession(sc, ses, enc, mac);
- if (error >= 0)
+ if (eu->sem_newsession(csp))
break;
-
eu++;
}
-
- /* If not found, return EINVAL */
- if (error < 0)
- return (EINVAL);
+ KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session"));
/* Save cipher key */
- if (enc && enc->cri_key) {
- ses->ss_klen = enc->cri_klen / 8;
- memcpy(ses->ss_key, enc->cri_key, ses->ss_klen);
- }
+ if (csp->csp_cipher_key != NULL)
+ memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen);
/* Save digest key */
- if (mac && mac->cri_key) {
- ses->ss_mklen = mac->cri_klen / 8;
- memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen);
+ if (csp->csp_auth_key != NULL)
+ memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen);
+
+ if (csp->csp_auth_alg != 0) {
+ if (csp->csp_auth_mlen == 0)
+ ses->ss_mlen = crypto_auth_hash(csp)->hashsize;
+ else
+ ses->ss_mlen = csp->csp_auth_mlen;
}
- ses->ss_eu = eu;
return (0);
}
@@ -1279,11 +1255,12 @@ sec_process(device_t dev, struct cryptop *crp, int hint)
{
struct sec_softc *sc = device_get_softc(dev);
struct sec_desc *desc = NULL;
- struct cryptodesc *mac, *enc;
+ const struct crypto_session_params *csp;
struct sec_session *ses;
- int buftype, error = 0;
+ int error = 0;
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
/* Check for input length */
if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) {
@@ -1292,13 +1269,6 @@ sec_process(device_t dev, struct cryptop *crp, int hint)
return (0);
}
- /* Get descriptors */
- if (sec_split_crp(crp, &enc, &mac)) {
- crp->crp_etype = EINVAL;
- crypto_done(crp);
- return (0);
- }
-
SEC_LOCK(sc, descriptors);
SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
@@ -1315,56 +1285,29 @@ sec_process(device_t dev, struct cryptop *crp, int hint)
desc->sd_error = 0;
desc->sd_crp = crp;
- if (crp->crp_flags & CRYPTO_F_IOV)
- buftype = SEC_UIO;
- else if (crp->crp_flags & CRYPTO_F_IMBUF)
- buftype = SEC_MBUF;
- else
- buftype = SEC_MEMORY;
-
- if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
- ses->ss_ivlen);
- else
- arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0);
-
- if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, ses->ss_ivlen,
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(desc->sd_desc->shd_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
desc->sd_desc->shd_iv);
- } else if (enc) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
- ses->ss_ivlen);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(desc->sd_desc->shd_iv, crp->crp_iv,
+ csp->csp_ivlen);
else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, ses->ss_ivlen,
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
desc->sd_desc->shd_iv);
}
- if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
- ses->ss_klen = enc->crd_klen / 8;
- memcpy(ses->ss_key, enc->crd_key, ses->ss_klen);
- } else
- error = E2BIG;
- }
+ if (crp->crp_cipher_key != NULL)
+ memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen);
- if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
- ses->ss_mklen = mac->crd_klen / 8;
- memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen);
- } else
- error = E2BIG;
- }
+ if (crp->crp_auth_key != NULL)
+ memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen);
- if (!error) {
- memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen);
- memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen);
+ memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen);
+ memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen);
- error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype);
- }
+ error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp);
if (error) {
SEC_DESC_FREE_POINTERS(desc);
@@ -1400,8 +1343,7 @@ sec_process(device_t dev, struct cryptop *crp, int hint)
static int
sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
- struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
- int buftype)
+ const struct crypto_session_params *csp, struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
int error;
@@ -1417,25 +1359,25 @@ sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
/* Pointer 1: IV IN */
error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
+ offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
if (error)
return (error);
/* Pointer 2: Cipher Key */
error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
+ offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
if (error)
return (error);
/* Pointer 3: Data IN */
- error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
/* Pointer 4: Data OUT */
- error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
@@ -1452,20 +1394,13 @@ sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
static int
sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
- struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
- struct cryptodesc *mac, int buftype)
+ const struct crypto_session_params *csp, struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
u_int eu, mode, hashlen;
int error;
- if (mac->crd_len < enc->crd_len)
- return (EINVAL);
-
- if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len)
- return (EINVAL);
-
- error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
+ error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
if (error)
return (error);
@@ -1475,144 +1410,107 @@ sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
/* Pointer 0: HMAC Key */
error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen);
+ offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen);
if (error)
return (error);
/* Pointer 1: HMAC-Only Data IN */
- error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip,
- mac->crd_len - enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start,
+ crp->crp_aad_length);
if (error)
return (error);
/* Pointer 2: Cipher Key */
error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
+ offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
if (error)
return (error);
/* Pointer 3: IV IN */
error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
+ offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
if (error)
return (error);
/* Pointer 4: Data IN */
- error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
/* Pointer 5: Data OUT */
- error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
/* Pointer 6: HMAC OUT */
- error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject,
- hashlen, buftype);
+ error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr +
+ offsetof(struct sec_hw_desc, shd_digest), hashlen);
return (error);
}
/* AESU */
-static int
-sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses,
- struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_aesu_newsession(const struct crypto_session_params *csp)
{
- if (enc == NULL)
- return (-1);
-
- if (enc->cri_alg != CRYPTO_AES_CBC)
- return (-1);
-
- ses->ss_ivlen = AES_BLOCK_LEN;
-
- return (0);
+ return (csp->csp_cipher_alg == CRYPTO_AES_CBC);
}
static int
-sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses,
- struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_aesu_make_desc(struct sec_softc *sc,
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
- struct cryptodesc *enc, *mac;
int error;
- error = sec_split_crp(crp, &enc, &mac);
- if (error)
- return (error);
-
- if (!enc)
- return (EINVAL);
-
hd->shd_eu_sel0 = SEC_EU_AESU;
hd->shd_mode0 = SEC_AESU_MODE_CBC;
- if (enc->crd_alg != CRYPTO_AES_CBC)
- return (EINVAL);
-
- if (enc->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
hd->shd_mode0 |= SEC_AESU_MODE_ED;
hd->shd_dir = 0;
} else
hd->shd_dir = 1;
- if (mac)
- error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
- buftype);
+ if (csp->csp_mode == CSP_MODE_ETA)
+ error = sec_build_common_s_desc(sc, desc, csp, crp);
else
- error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
- buftype);
+ error = sec_build_common_ns_desc(sc, desc, csp, crp);
return (error);
}
/* DEU */
-static int
-sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses,
- struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_deu_newsession(const struct crypto_session_params *csp)
{
- if (enc == NULL)
- return (-1);
-
- switch (enc->cri_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_DES_CBC:
case CRYPTO_3DES_CBC:
- break;
+ return (true);
default:
- return (-1);
+ return (false);
}
-
- ses->ss_ivlen = DES_BLOCK_LEN;
-
- return (0);
}
static int
-sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
- struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_deu_make_desc(struct sec_softc *sc, const struct crypto_session_params *csp,
+ struct sec_desc *desc, struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
- struct cryptodesc *enc, *mac;
int error;
- error = sec_split_crp(crp, &enc, &mac);
- if (error)
- return (error);
-
- if (!enc)
- return (EINVAL);
-
hd->shd_eu_sel0 = SEC_EU_DEU;
hd->shd_mode0 = SEC_DEU_MODE_CBC;
- switch (enc->crd_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_3DES_CBC:
hd->shd_mode0 |= SEC_DEU_MODE_TS;
break;
@@ -1622,25 +1520,23 @@ sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
return (EINVAL);
}
- if (enc->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
hd->shd_mode0 |= SEC_DEU_MODE_ED;
hd->shd_dir = 0;
} else
hd->shd_dir = 1;
- if (mac)
- error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
- buftype);
+ if (csp->csp_mode == CSP_MODE_ETA)
+ error = sec_build_common_s_desc(sc, desc, csp, crp);
else
- error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
- buftype);
+ error = sec_build_common_ns_desc(sc, desc, csp, crp);
return (error);
}
/* MDEU */
-static int
+static bool
sec_mdeu_can_handle(u_int alg)
{
switch (alg) {
@@ -1651,20 +1547,21 @@ sec_mdeu_can_handle(u_int alg)
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
- return (1);
+ return (true);
default:
- return (0);
+ return (false);
}
}
static int
-sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
+sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode,
+ u_int *hashlen)
{
*mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
*eu = SEC_EU_NONE;
- switch (crd->crd_alg) {
+ switch (csp->csp_auth_alg) {
case CRYPTO_MD5_HMAC:
*mode |= SEC_MDEU_MODE_HMAC;
/* FALLTHROUGH */
@@ -1703,34 +1600,23 @@ sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
return (0);
}
-static int
-sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses,
- struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_mdeu_newsession(const struct crypto_session_params *csp)
{
- if (mac && sec_mdeu_can_handle(mac->cri_alg))
- return (0);
-
- return (-1);
+ return (sec_mdeu_can_handle(csp->csp_auth_alg));
}
static int
-sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
- struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_mdeu_make_desc(struct sec_softc *sc,
+ const struct crypto_session_params *csp,
+ struct sec_desc *desc, struct cryptop *crp)
{
- struct cryptodesc *enc, *mac;
struct sec_hw_desc *hd = desc->sd_desc;
u_int eu, mode, hashlen;
int error;
- error = sec_split_crp(crp, &enc, &mac);
- if (error)
- return (error);
-
- if (enc)
- return (EINVAL);
-
- error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
+ error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
if (error)
return (error);
@@ -1754,7 +1640,7 @@ sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
error = sec_make_pointer_direct(sc, desc, 2,
desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
- shd_mkey), ses->ss_mklen);
+ shd_mkey), csp->csp_auth_klen);
else
error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
@@ -1762,8 +1648,8 @@ sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
return (error);
/* Pointer 3: Input Data */
- error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip,
- mac->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
@@ -1773,8 +1659,8 @@ sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
return (error);
/* Pointer 5: Hash out */
- error = sec_make_pointer(sc, desc, 5, crp->crp_buf,
- mac->crd_inject, hashlen, buftype);
+ error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr +
+ offsetof(struct sec_hw_desc, shd_digest), hashlen);
if (error)
return (error);
diff --git a/sys/dev/sec/sec.h b/sys/dev/sec/sec.h
index 05b15039ad64..6ad482316a54 100644
--- a/sys/dev/sec/sec.h
+++ b/sys/dev/sec/sec.h
@@ -98,6 +98,7 @@ struct sec_hw_desc {
uint8_t shd_iv[SEC_MAX_IV_LEN];
uint8_t shd_key[SEC_MAX_KEY_LEN];
uint8_t shd_mkey[SEC_MAX_KEY_LEN];
+ uint8_t shd_digest[HASH_MAX_LEN];
} __packed__;
#define shd_eu_sel0 shd_control.request.eu_sel0
@@ -144,21 +145,17 @@ struct sec_lt {
};
struct sec_eu_methods {
- int (*sem_newsession)(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc,
- struct cryptoini *mac);
+ bool (*sem_newsession)(const struct crypto_session_params *csp);
int (*sem_make_desc)(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc,
- struct cryptop *crp, int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
};
struct sec_session {
struct sec_eu_methods *ss_eu;
uint8_t ss_key[SEC_MAX_KEY_LEN];
uint8_t ss_mkey[SEC_MAX_KEY_LEN];
- u_int ss_klen;
- u_int ss_mklen;
- u_int ss_ivlen;
+ int ss_mlen;
};
struct sec_desc_map_info {
@@ -319,11 +316,6 @@ struct sec_softc {
(((sc)->sc_lt_free_cnt - (sc)->sc_lt_alloc_cnt - 1) \
& (SEC_LT_ENTRIES - 1))
-/* DMA Maping defines */
-#define SEC_MEMORY 0
-#define SEC_UIO 1
-#define SEC_MBUF 2
-
/* Size of SEC registers area */
#define SEC_IO_SIZE 0x10000
diff --git a/sys/dev/ubsec/ubsec.c b/sys/dev/ubsec/ubsec.c
index 19f46458ac3b..e4b324e05f86 100644
--- a/sys/dev/ubsec/ubsec.c
+++ b/sys/dev/ubsec/ubsec.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
+#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -70,10 +71,8 @@ __FBSDID("$FreeBSD$");
#include <sys/bus.h>
#include <sys/rman.h>
-#include <crypto/sha1.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
-#include <sys/md5.h>
+#include <opencrypto/xform_auth.h>
#include <sys/random.h>
#include <sys/kobj.h>
@@ -111,7 +110,9 @@ static int ubsec_suspend(device_t);
static int ubsec_resume(device_t);
static int ubsec_shutdown(device_t);
-static int ubsec_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int ubsec_probesession(device_t, const struct crypto_session_params *);
+static int ubsec_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int ubsec_process(device_t, struct cryptop *, int);
static int ubsec_kprocess(device_t, struct cryptkop *, int);
@@ -125,6 +126,7 @@ static device_method_t ubsec_methods[] = {
DEVMETHOD(device_shutdown, ubsec_shutdown),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, ubsec_probesession),
DEVMETHOD(cryptodev_newsession, ubsec_newsession),
DEVMETHOD(cryptodev_process, ubsec_process),
DEVMETHOD(cryptodev_kprocess, ubsec_kprocess),
@@ -348,13 +350,6 @@ ubsec_attach(device_t dev)
goto bad2;
}
- sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session),
- CRYPTOCAP_F_HARDWARE);
- if (sc->sc_cid < 0) {
- device_printf(dev, "could not get crypto driver id\n");
- goto bad3;
- }
-
/*
* Setup DMA descriptor area.
*/
@@ -370,7 +365,7 @@ ubsec_attach(device_t dev)
NULL, NULL, /* lockfunc, lockarg */
&sc->sc_dmat)) {
device_printf(dev, "cannot allocate DMA tag\n");
- goto bad4;
+ goto bad3;
}
SIMPLEQ_INIT(&sc->sc_freequeue);
dmap = sc->sc_dmaa;
@@ -404,11 +399,6 @@ ubsec_attach(device_t dev)
device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-
/*
* Reset Broadcom chip
*/
@@ -424,6 +414,13 @@ ubsec_attach(device_t dev)
*/
ubsec_init_board(sc);
+ sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session),
+ CRYPTOCAP_F_HARDWARE);
+ if (sc->sc_cid < 0) {
+ device_printf(dev, "could not get crypto driver id\n");
+ goto bad4;
+ }
+
#ifndef UBSEC_NO_RNG
if (sc->sc_flags & UBS_FLAGS_RNG) {
sc->sc_statmask |= BS_STAT_MCR2_DONE;
@@ -477,7 +474,15 @@ skip_rng:
}
return (0);
bad4:
- crypto_unregister_all(sc->sc_cid);
+ while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
+ struct ubsec_q *q;
+
+ q = SIMPLEQ_FIRST(&sc->sc_freequeue);
+ SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
+ ubsec_dma_free(sc, &q->q_dma->d_alloc);
+ free(q, M_DEVBUF);
+ }
+ bus_dma_tag_destroy(sc->sc_dmat);
bad3:
bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bad2:
@@ -498,13 +503,14 @@ ubsec_detach(device_t dev)
/* XXX wait/abort active ops */
+ crypto_unregister_all(sc->sc_cid);
+
/* disable interrupts */
WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
callout_stop(&sc->sc_rngto);
-
- crypto_unregister_all(sc->sc_cid);
+ bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
#ifdef UBSEC_RNDTEST
if (sc->sc_rndtest)
@@ -531,7 +537,6 @@ ubsec_detach(device_t dev)
mtx_destroy(&sc->sc_mcr2lock);
bus_generic_detach(dev);
- bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bus_dma_tag_destroy(sc->sc_dmat);
@@ -826,7 +831,7 @@ feed1:
}
static void
-ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
+ubsec_setup_enckey(struct ubsec_session *ses, int algo, const void *key)
{
/* Go ahead and compute key in ubsec's byte order */
@@ -846,112 +851,134 @@ ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
}
static void
-ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
+ubsec_setup_mackey(struct ubsec_session *ses, int algo, const char *key,
+ int klen)
{
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
- int i;
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
+
+ hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx);
+ bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
+
+ explicit_bzero(&md5ctx, sizeof(md5ctx));
} else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_ipad_buffer,
- SHA1_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
+
+ hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
+ bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
+
+ explicit_bzero(&sha1ctx, sizeof(sha1ctx));
}
+}
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
+static bool
+ubsec_auth_supported(const struct crypto_session_params *csp)
+{
- if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen);
- bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
- } else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_opad_buffer,
- SHA1_BLOCK_LEN - klen);
- bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ return (true);
+ default:
+ return (false);
}
+}
+
+static bool
+ubsec_cipher_supported(const struct crypto_session_params *csp)
+{
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ return (csp->csp_ivlen == 8);
+ default:
+ return (false);
+ }
}
-/*
- * Allocate a new 'session' and return an encoded session id. 'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
- */
static int
-ubsec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ubsec_probesession(device_t dev, const struct crypto_session_params *csp)
{
- struct ubsec_softc *sc = device_get_softc(dev);
- struct cryptoini *c, *encini = NULL, *macini = NULL;
- struct ubsec_session *ses = NULL;
- if (cri == NULL || sc == NULL)
+ if (csp->csp_flags != 0)
return (EINVAL);
-
- for (c = cri; c != NULL; c = c->cri_next) {
- if (c->cri_alg == CRYPTO_MD5_HMAC ||
- c->cri_alg == CRYPTO_SHA1_HMAC) {
- if (macini)
- return (EINVAL);
- macini = c;
- } else if (c->cri_alg == CRYPTO_DES_CBC ||
- c->cri_alg == CRYPTO_3DES_CBC) {
- if (encini)
- return (EINVAL);
- encini = c;
- } else
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!ubsec_auth_supported(csp))
return (EINVAL);
- }
- if (encini == NULL && macini == NULL)
+ break;
+ case CSP_MODE_CIPHER:
+ if (!ubsec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!ubsec_auth_supported(csp) ||
+ !ubsec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
return (EINVAL);
+ }
- ses = crypto_get_driver_session(cses);
- if (encini) {
- /* get an IV, network byte order */
- /* XXX may read fewer than requested */
- read_random(ses->ses_iv, sizeof(ses->ses_iv));
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
- if (encini->cri_key != NULL) {
- ubsec_setup_enckey(ses, encini->cri_alg,
- encini->cri_key);
- }
- }
+/*
+ * Allocate a new 'session'.
+ */
+static int
+ubsec_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct ubsec_session *ses;
+
+ ses = crypto_get_driver_session(cses);
+ if (csp->csp_cipher_alg != 0 && csp->csp_cipher_key != NULL)
+ ubsec_setup_enckey(ses, csp->csp_cipher_alg,
+ csp->csp_cipher_key);
- if (macini) {
- ses->ses_mlen = macini->cri_mlen;
+ if (csp->csp_auth_alg != 0) {
+ ses->ses_mlen = csp->csp_auth_mlen;
if (ses->ses_mlen == 0) {
- if (macini->cri_alg == CRYPTO_MD5_HMAC)
+ if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
ses->ses_mlen = MD5_HASH_LEN;
else
ses->ses_mlen = SHA1_HASH_LEN;
}
- if (macini->cri_key != NULL) {
- ubsec_setup_mackey(ses, macini->cri_alg,
- macini->cri_key, macini->cri_klen / 8);
+ if (csp->csp_auth_key != NULL) {
+ ubsec_setup_mackey(ses, csp->csp_auth_alg,
+ csp->csp_auth_key, csp->csp_auth_klen);
}
}
return (0);
}
+static bus_size_t
+ubsec_crp_length(struct cryptop *crp)
+{
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (crp->crp_mbuf->m_pkthdr.len);
+ case CRYPTO_BUF_UIO:
+ return (crp->crp_uio->uio_resid);
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_ilen);
+ default:
+ panic("bad crp buffer type");
+ }
+}
+
static void
-ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
{
struct ubsec_operand *op = arg;
@@ -959,12 +986,11 @@ ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, in
("Too many DMA segments returned when mapping operand"));
#ifdef UBSEC_DEBUG
if (ubsec_debug)
- printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
- (u_int) mapsize, nsegs, error);
+ printf("ubsec_op_cb: nsegs %d error %d\n",
+ nsegs, error);
#endif
if (error != 0)
return;
- op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
@@ -972,22 +998,17 @@ ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, in
static int
ubsec_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct ubsec_softc *sc = device_get_softc(dev);
struct ubsec_q *q = NULL;
int err = 0, i, j, nicealign;
- struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
- int encoffset = 0, macoffset = 0, cpskip, cpoffset;
+ int cpskip, cpoffset;
int sskip, dskip, stheend, dtheend;
int16_t coffset;
struct ubsec_session *ses;
struct ubsec_pktctx ctx;
struct ubsec_dma *dmap = NULL;
- if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
- ubsecstats.hst_invalid++;
- return (EINVAL);
- }
-
mtx_lock(&sc->sc_freeqlock);
if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
ubsecstats.hst_queuefull++;
@@ -1006,103 +1027,34 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
q->q_dma = dmap;
ses = crypto_get_driver_session(crp->crp_session);
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- q->q_src_m = (struct mbuf *)crp->crp_buf;
- q->q_dst_m = (struct mbuf *)crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- q->q_src_io = (struct uio *)crp->crp_buf;
- q->q_dst_io = (struct uio *)crp->crp_buf;
- } else {
- ubsecstats.hst_badflags++;
- err = EINVAL;
- goto errout; /* XXX we don't handle contiguous blocks! */
- }
-
bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
dmap->d_dma->d_mcr.mcr_flags = 0;
q->q_crp = crp;
- crd1 = crp->crp_desc;
- if (crd1 == NULL) {
- ubsecstats.hst_nodesc++;
- err = EINVAL;
- goto errout;
- }
- crd2 = crd1->crd_next;
+ csp = crypto_get_params(crp->crp_session);
- if (crd2 == NULL) {
- if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC) {
- maccrd = crd1;
- enccrd = NULL;
- } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC) {
- maccrd = NULL;
- enccrd = crd1;
- } else {
- ubsecstats.hst_badalg++;
- err = EINVAL;
- goto errout;
- }
- } else {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC) &&
- ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
- maccrd = crd1;
- enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
- (crd1->crd_flags & CRD_F_ENCRYPT)) {
- enccrd = crd1;
- maccrd = crd2;
- } else {
- /*
- * We cannot order the ubsec as requested
- */
- ubsecstats.hst_badalg++;
- err = EINVAL;
- goto errout;
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_cipher_key != NULL) {
+ ubsec_setup_enckey(ses, csp->csp_cipher_alg,
+ crp->crp_cipher_key);
}
- }
- if (enccrd) {
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- ubsec_setup_enckey(ses, enccrd->crd_alg,
- enccrd->crd_key);
- }
-
- encoffset = enccrd->crd_skip;
ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
- q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
-
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
- else {
- ctx.pc_iv[0] = ses->ses_iv[0];
- ctx.pc_iv[1] = ses->ses_iv[1];
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(ctx.pc_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start,
+ csp->csp_ivlen, ctx.pc_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(ctx.pc_iv, crp->crp_iv, csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ ctx.pc_iv);
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
- }
- } else {
+ if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
-
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
- else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
- }
}
ctx.pc_deskey[0] = ses->ses_deskey[0];
@@ -1115,15 +1067,13 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
SWAP32(ctx.pc_iv[1]);
}
- if (maccrd) {
- if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- ubsec_setup_mackey(ses, maccrd->crd_alg,
- maccrd->crd_key, maccrd->crd_klen / 8);
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_auth_key != NULL) {
+ ubsec_setup_mackey(ses, csp->csp_auth_alg,
+ crp->crp_auth_key, csp->csp_auth_klen);
}
- macoffset = maccrd->crd_skip;
-
- if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
+ if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
else
ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
@@ -1137,35 +1087,37 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
}
}
- if (enccrd && maccrd) {
+ if (csp->csp_mode == CSP_MODE_ETA) {
/*
- * ubsec cannot handle packets where the end of encryption
- * and authentication are not the same, or where the
- * encrypted part begins before the authenticated part.
+ * ubsec only supports ETA requests where there is no
+ * gap between the AAD and payload.
*/
- if ((encoffset + enccrd->crd_len) !=
- (macoffset + maccrd->crd_len)) {
+ if (crp->crp_aad_length != 0 &&
+ crp->crp_aad_start + crp->crp_aad_length !=
+ crp->crp_payload_start) {
ubsecstats.hst_lenmismatch++;
err = EINVAL;
goto errout;
}
- if (enccrd->crd_skip < maccrd->crd_skip) {
- ubsecstats.hst_skipmismatch++;
- err = EINVAL;
- goto errout;
+
+ if (crp->crp_aad_length != 0) {
+ sskip = crp->crp_aad_start;
+ } else {
+ sskip = crp->crp_payload_start;
}
- sskip = maccrd->crd_skip;
- cpskip = dskip = enccrd->crd_skip;
- stheend = maccrd->crd_len;
- dtheend = enccrd->crd_len;
- coffset = enccrd->crd_skip - maccrd->crd_skip;
+ cpskip = dskip = crp->crp_payload_start;
+ stheend = crp->crp_aad_length + crp->crp_payload_length;
+ dtheend = crp->crp_payload_length;
+ coffset = crp->crp_aad_length;
cpoffset = cpskip + dtheend;
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
- printf("mac: skip %d, len %d, inject %d\n",
- maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
- printf("enc: skip %d, len %d, inject %d\n",
- enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
+ printf("AAD: start %d, len %d, digest %d\n",
+ crp->crp_aad_start, crp->crp_aad_length,
+ crp->crp_digest_start);
+ printf("payload: start %d, len %d, IV %d\n",
+ crp->crp_payload_start, crp->crp_payload_length,
+ crp->crp_iv_start);
printf("src: skip %d, len %d\n", sskip, stheend);
printf("dst: skip %d, len %d\n", dskip, dtheend);
printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
@@ -1173,8 +1125,8 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
}
#endif
} else {
- cpskip = dskip = sskip = macoffset + encoffset;
- dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
+ cpskip = dskip = sskip = crp->crp_payload_start;
+ dtheend = stheend = crp->crp_payload_length;
cpoffset = cpskip + dtheend;
coffset = 0;
}
@@ -1185,25 +1137,15 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
err = ENOMEM;
goto errout;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
- q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
- q->q_src_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
- q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
- q->q_src_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
+ if (bus_dmamap_load_crp(sc->sc_dmat, q->q_src_map, crp, ubsec_op_cb,
+ &q->q_src, BUS_DMA_NOWAIT) != 0) {
+ bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
+ q->q_src_map = NULL;
+ ubsecstats.hst_noload++;
+ err = ENOMEM;
+ goto errout;
}
+ q->q_src_mapsize = ubsec_crp_length(crp);
nicealign = ubsec_dmamap_aligned(&q->q_src);
dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
@@ -1257,7 +1199,7 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
j++;
}
- if (enccrd == NULL && maccrd != NULL) {
+ if (csp->csp_mode == CSP_MODE_DIGEST) {
dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
@@ -1270,104 +1212,79 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
#endif
} else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- if (!nicealign) {
- ubsecstats.hst_iovmisaligned++;
- err = EINVAL;
- goto errout;
+ if (nicealign) {
+ q->q_dst = q->q_src;
+ } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ int totlen, len;
+ struct mbuf *m, *top, **mp;
+
+ ubsecstats.hst_unaligned++;
+ totlen = q->q_src_mapsize;
+ if (totlen >= MINCLSIZE) {
+ m = m_getcl(M_NOWAIT, MT_DATA,
+ crp->crp_mbuf->m_flags & M_PKTHDR);
+ len = MCLBYTES;
+ } else if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+ m = m_gethdr(M_NOWAIT, MT_DATA);
+ len = MHLEN;
+ } else {
+ m = m_get(M_NOWAIT, MT_DATA);
+ len = MLEN;
}
- if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
- &q->q_dst_map)) {
- ubsecstats.hst_nomap++;
- err = ENOMEM;
- goto errout;
+ if (m && crp->crp_mbuf->m_flags & M_PKTHDR &&
+ !m_dup_pkthdr(m, crp->crp_mbuf, M_NOWAIT)) {
+ m_free(m);
+ m = NULL;
}
- if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
- q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
- q->q_dst_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
+ if (m == NULL) {
+ ubsecstats.hst_nombuf++;
+ err = sc->sc_nqueue ? ERESTART : ENOMEM;
goto errout;
}
- } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (nicealign) {
- q->q_dst = q->q_src;
- } else {
- int totlen, len;
- struct mbuf *m, *top, **mp;
+ m->m_len = len = min(totlen, len);
+ totlen -= len;
+ top = m;
+ mp = &top;
- ubsecstats.hst_unaligned++;
- totlen = q->q_src_mapsize;
+ while (totlen > 0) {
if (totlen >= MINCLSIZE) {
- m = m_getcl(M_NOWAIT, MT_DATA,
- q->q_src_m->m_flags & M_PKTHDR);
+ m = m_getcl(M_NOWAIT, MT_DATA, 0);
len = MCLBYTES;
- } else if (q->q_src_m->m_flags & M_PKTHDR) {
- m = m_gethdr(M_NOWAIT, MT_DATA);
- len = MHLEN;
} else {
m = m_get(M_NOWAIT, MT_DATA);
len = MLEN;
}
- if (m && q->q_src_m->m_flags & M_PKTHDR &&
- !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
- m_free(m);
- m = NULL;
- }
if (m == NULL) {
+ m_freem(top);
ubsecstats.hst_nombuf++;
err = sc->sc_nqueue ? ERESTART : ENOMEM;
goto errout;
}
m->m_len = len = min(totlen, len);
totlen -= len;
- top = m;
- mp = &top;
-
- while (totlen > 0) {
- if (totlen >= MINCLSIZE) {
- m = m_getcl(M_NOWAIT,
- MT_DATA, 0);
- len = MCLBYTES;
- } else {
- m = m_get(M_NOWAIT, MT_DATA);
- len = MLEN;
- }
- if (m == NULL) {
- m_freem(top);
- ubsecstats.hst_nombuf++;
- err = sc->sc_nqueue ? ERESTART : ENOMEM;
- goto errout;
- }
- m->m_len = len = min(totlen, len);
- totlen -= len;
- *mp = m;
- mp = &m->m_next;
- }
- q->q_dst_m = top;
- ubsec_mcopy(q->q_src_m, q->q_dst_m,
- cpskip, cpoffset);
- if (bus_dmamap_create(sc->sc_dmat,
- BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
- ubsecstats.hst_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_mbuf(sc->sc_dmat,
- q->q_dst_map, q->q_dst_m,
- ubsec_op_cb, &q->q_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat,
- q->q_dst_map);
- q->q_dst_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
+ *mp = m;
+ mp = &m->m_next;
+ }
+ q->q_dst_m = top;
+ ubsec_mcopy(crp->crp_mbuf, q->q_dst_m, cpskip, cpoffset);
+ if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
+ &q->q_dst_map) != 0) {
+ ubsecstats.hst_nomap++;
+ err = ENOMEM;
+ goto errout;
+ }
+ if (bus_dmamap_load_mbuf_sg(sc->sc_dmat,
+ q->q_dst_map, q->q_dst_m, q->q_dst_segs,
+ &q->q_dst_nsegs, 0) != 0) {
+ bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
+ q->q_dst_map = NULL;
+ ubsecstats.hst_noload++;
+ err = ENOMEM;
+ goto errout;
}
+ q->q_dst_mapsize = q->q_src_mapsize;
} else {
- ubsecstats.hst_badflags++;
+ ubsecstats.hst_iovmisaligned++;
err = EINVAL;
goto errout;
}
@@ -1414,7 +1331,7 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
pb->pb_len = htole32(packl);
if ((i + 1) == q->q_dst_nsegs) {
- if (maccrd)
+ if (csp->csp_auth_alg != 0)
pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_macbuf[0]));
else
@@ -1465,7 +1382,7 @@ ubsec_process(device_t dev, struct cryptop *crp, int hint)
errout:
if (q != NULL) {
- if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
+ if (q->q_dst_m != NULL)
m_freem(q->q_dst_m);
if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
@@ -1495,12 +1412,14 @@ errout:
static void
ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
{
+ const struct crypto_session_params *csp;
struct cryptop *crp = (struct cryptop *)q->q_crp;
struct ubsec_session *ses;
- struct cryptodesc *crd;
struct ubsec_dma *dmap = q->q_dma;
+ char hash[SHA1_HASH_LEN];
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
ubsecstats.hst_opackets++;
ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
@@ -1517,31 +1436,21 @@ ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
- if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
- m_freem(q->q_src_m);
- crp->crp_buf = (caddr_t)q->q_dst_m;
- }
-
- /* copy out IV for future use */
- if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (crd->crd_alg != CRYPTO_DES_CBC &&
- crd->crd_alg != CRYPTO_3DES_CBC)
- continue;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + crd->crd_len - 8, 8,
- (caddr_t)ses->ses_iv);
- break;
- }
+ if (q->q_dst_m != NULL) {
+ m_freem(crp->crp_mbuf);
+ crp->crp_mbuf = q->q_dst_m;
}
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (crd->crd_alg != CRYPTO_MD5_HMAC &&
- crd->crd_alg != CRYPTO_SHA1_HMAC)
- continue;
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- ses->ses_mlen, (caddr_t)dmap->d_dma->d_macbuf);
- break;
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ ses->ses_mlen, hash);
+ if (timingsafe_bcmp(dmap->d_dma->d_macbuf, hash,
+ ses->ses_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ ses->ses_mlen, dmap->d_dma->d_macbuf);
}
mtx_lock(&sc->sc_freeqlock);
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
@@ -1942,7 +1851,7 @@ ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
if(q->q_stacked_mcr[i]) {
q2 = q->q_stacked_mcr[i];
- if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
+ if (q2->q_dst_m != NULL)
m_freem(q2->q_dst_m);
crp = (struct cryptop *)q2->q_crp;
@@ -1959,7 +1868,7 @@ ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
/*
* Free header MCR
*/
- if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
+ if (q->q_dst_m != NULL)
m_freem(q->q_dst_m);
crp = (struct cryptop *)q->q_crp;
diff --git a/sys/dev/ubsec/ubsecvar.h b/sys/dev/ubsec/ubsecvar.h
index ae6d5e2cb6bc..b857061be5c0 100644
--- a/sys/dev/ubsec/ubsecvar.h
+++ b/sys/dev/ubsec/ubsecvar.h
@@ -134,10 +134,6 @@ struct ubsec_dma {
#define UBS_FLAGS_RNG 0x10 /* hardware rng */
struct ubsec_operand {
- union {
- struct mbuf *m;
- struct uio *io;
- } u;
bus_dmamap_t map;
bus_size_t mapsize;
int nsegs;
@@ -153,19 +149,16 @@ struct ubsec_q {
struct ubsec_operand q_src;
struct ubsec_operand q_dst;
+ struct mbuf *q_dst_m;
int q_flags;
};
-#define q_src_m q_src.u.m
-#define q_src_io q_src.u.io
#define q_src_map q_src.map
#define q_src_nsegs q_src.nsegs
#define q_src_segs q_src.segs
#define q_src_mapsize q_src.mapsize
-#define q_dst_m q_dst.u.m
-#define q_dst_io q_dst.u.io
#define q_dst_map q_dst.map
#define q_dst_nsegs q_dst.nsegs
#define q_dst_segs q_dst.segs
@@ -215,7 +208,6 @@ struct ubsec_session {
u_int32_t ses_mlen; /* hmac length */
u_int32_t ses_hminner[5]; /* hmac inner state */
u_int32_t ses_hmouter[5]; /* hmac outer state */
- u_int32_t ses_iv[2]; /* [3]DES iv */
};
#endif /* _KERNEL */