Add qat(4)

This provides an OpenCrypto driver for Intel QuickAssist devices.  The
driver was initially ported from NetBSD and comes with a few
improvements:
- support for GMAC/AES-GCM, AES-CTR and AES-XTS, and support for
  SHA/HMAC-authenticated encryption
- support for detaching the driver
- various bug fixes
- DH895X support

Discussed with:	jhb
MFC after:	3 days
Sponsored by:	Rubicon Communications, LLC (Netgate)
Differential Revision:	https://reviews.freebsd.org/D26963

27 files changed, 15644 insertions, 0 deletions

diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index efc4d64c62ca..c4647c7d4b56 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -431,6 +431,7 @@ MAN=	aac.4 \
 	pty.4 \
 	puc.4 \
 	pwmc.4 \
+	${_qat.4} \
 	${_qlxge.4} \
 	${_qlxgb.4} \
 	${_qlxgbe.4} \
@@ -823,6 +824,7 @@ _nvram.4=	nvram.4
 _ossl.4=	ossl.4
 _padlock.4=	padlock.4
 _pchtherm.4=	pchtherm.4
+_qat.4=		qat.4
 _rr232x.4=	rr232x.4
 _speaker.4=	speaker.4
 _spkr.4=	spkr.4
diff --git a/share/man/man4/qat.4 b/share/man/man4/qat.4
new file mode 100644
index 000000000000..9e9491f22aea
--- /dev/null
+++ b/share/man/man4/qat.4
@@ -0,0 +1,99 @@
+.\"-
+.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate)
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd November 5, 2020
+.Dt QAT 4
+.Os
+.Sh NAME
+.Nm qat
+.Nd Intel QuickAssist Technology (QAT) driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device crypto"
+.Cd "device cryptodev"
+.Cd "device qat"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following lines in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+qat_load="YES"
+qat_c2xxxfw_load="YES"
+qat_c3xxxfw_load="YES"
+qat_c63xfw_load="YES"
+qat_d15xxfw_load="YES"
+qat_dh895xcc_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+driver implements
+.Xr crypto 4
+support for some of the cryptographic acceleration functions of the Intel
+QuickAssist device.
+The
+.Nm
+driver supports the QAT devices integrated with Atom C2000 and C3000 and Xeon
+C620 and D-1500 chipsets, and the Intel QAT Adapter 8950.
+It can accelerate AES in CBC, CTR, XTS (except for the C2000) and GCM modes,
+and can perform authenticated encryption combining the CBC, CTR and XTS modes
+with SHA1-HMAC and SHA2-HMAC.
+The
+.Nm
+driver can also compute SHA1 and SHA2 digests.
+.Sh SEE ALSO
+.Xr crypto 4 ,
+.Xr ipsec 4 ,
+.Xr pci 4 ,
+.Xr random 4 ,
+.Xr crypto 7 ,
+.Xr crypto 9
+.Sh HISTORY
+The
+.Nm
+driver first appeared in
+.Fx 13.0 .
+.Sh AUTHORS
+The
+.Nm
+driver was written for
+.Nx
+by
+.An Hikaru Abe Aq Mt hikaru@iij.ad.jp
+and ported to
+.Fx
+by
+.An Mark Johnston Aq Mt markj@FreeBSD.org .
+.Sh BUGS
+Some Atom C2000 QAT devices have two acceleration engines instead of one.
+The
+.Nm
+driver currently misbehaves when both are enabled and thus does not enable
+the second acceleration engine if one is present.
diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
index 4bee0f804970..4ee84a591fc8 100644
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -471,6 +471,10 @@ device		vmd_bus		# bus for VMD children
 device		pmspcv
 
 #
+# Intel QuickAssist
+device		qat
+
+#
 # SafeNet crypto driver: can be moved to the MI NOTES as soon as
 # it's tested on a big-endian machine
 #
diff --git a/sys/conf/files.x86 b/sys/conf/files.x86
index 1ac490b46517..968807bd8950 100644
--- a/sys/conf/files.x86
+++ b/sys/conf/files.x86
@@ -291,6 +291,15 @@ dev/isci/scil/scif_sas_task_request_state_handlers.c		optional isci
 dev/isci/scil/scif_sas_task_request_states.c			optional isci
 dev/isci/scil/scif_sas_timer.c					optional isci
 dev/itwd/itwd.c							optional itwd
+dev/qat/qat.c			optional qat
+dev/qat/qat_ae.c		optional qat
+dev/qat/qat_c2xxx.c		optional qat
+dev/qat/qat_c3xxx.c		optional qat
+dev/qat/qat_c62x.c		optional qat
+dev/qat/qat_d15xx.c		optional qat
+dev/qat/qat_dh895xcc.c		optional qat
+dev/qat/qat_hw15.c		optional qat
+dev/qat/qat_hw17.c		optional qat
 libkern/x86/crc32_sse42.c	standard
 #
 # x86 shared code between IA32 and AMD64 architectures
diff --git a/sys/dev/qat/qat.c b/sys/dev/qat/qat.c
new file mode 100644
index 000000000000..de07f8db5642
--- /dev/null
+++ b/sys/dev/qat/qat.c
@@ -0,0 +1,2140 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/firmware.h>
+#include <sys/kernel.h>
+#include <sys/mbuf.h>
+#include <sys/md5.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/rman.h>
+
+#include <machine/bus.h>
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform.h>
+
+#include "cryptodev_if.h"
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qatvar.h"
+#include "qat_aevar.h"
+
+extern struct qat_hw qat_hw_c2xxx;
+extern struct qat_hw qat_hw_c3xxx;
+extern struct qat_hw qat_hw_c62x;
+extern struct qat_hw qat_hw_d15xx;
+extern struct qat_hw qat_hw_dh895xcc;
+
+#define PCI_VENDOR_INTEL			0x8086
+#define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS	0x1f18
+#define PCI_PRODUCT_INTEL_C3K_QAT		0x19e2
+#define PCI_PRODUCT_INTEL_C3K_QAT_VF		0x19e3
+#define PCI_PRODUCT_INTEL_C620_QAT		0x37c8
+#define PCI_PRODUCT_INTEL_C620_QAT_VF		0x37c9
+#define PCI_PRODUCT_INTEL_XEOND_QAT		0x6f54
+#define PCI_PRODUCT_INTEL_XEOND_QAT_VF		0x6f55
+#define PCI_PRODUCT_INTEL_DH895XCC_QAT		0x0435
+#define PCI_PRODUCT_INTEL_DH895XCC_QAT_VF	0x0443
+
+static const struct qat_product {
+	uint16_t qatp_vendor;
+	uint16_t qatp_product;
+	const char *qatp_name;
+	enum qat_chip_type qatp_chip;
+	const struct qat_hw *qatp_hw;
+} qat_products[] = {
+	{ PCI_VENDOR_INTEL,	PCI_PRODUCT_INTEL_C2000_IQIA_PHYS,
+	  "Intel C2000 QuickAssist PF",
+	  QAT_CHIP_C2XXX, &qat_hw_c2xxx },
+	{ PCI_VENDOR_INTEL,	PCI_PRODUCT_INTEL_C3K_QAT,
+	  "Intel C3000 QuickAssist PF",
+	  QAT_CHIP_C3XXX, &qat_hw_c3xxx },
+	{ PCI_VENDOR_INTEL,	PCI_PRODUCT_INTEL_C620_QAT,
+	  "Intel C620/Xeon D-2100 QuickAssist PF",
+	  QAT_CHIP_C62X, &qat_hw_c62x },
+	{ PCI_VENDOR_INTEL,	PCI_PRODUCT_INTEL_XEOND_QAT,
+	  "Intel Xeon D-1500 QuickAssist PF",
+	  QAT_CHIP_D15XX, &qat_hw_d15xx },
+	{ PCI_VENDOR_INTEL,	PCI_PRODUCT_INTEL_DH895XCC_QAT,
+	  "Intel 8950 QuickAssist PCIe Adapter PF",
+	  QAT_CHIP_DH895XCC, &qat_hw_dh895xcc },
+	{ 0, 0, NULL, 0, NULL },
+};
+
+/* Hash Algorithm specific structure */
+
+/* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = {
+	0x67, 0x45, 0x23, 0x01,
+	0xef, 0xcd, 0xab, 0x89,
+	0x98, 0xba, 0xdc, 0xfe,
+	0x10, 0x32, 0x54, 0x76,
+	0xc3, 0xd2, 0xe1, 0xf0
+};
+
+/* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = {
+	0x6a, 0x09, 0xe6, 0x67,
+	0xbb, 0x67, 0xae, 0x85,
+	0x3c, 0x6e, 0xf3, 0x72,
+	0xa5, 0x4f, 0xf5, 0x3a,
+	0x51, 0x0e, 0x52, 0x7f,
+	0x9b, 0x05, 0x68, 0x8c,
+	0x1f, 0x83, 0xd9, 0xab,
+	0x5b, 0xe0, 0xcd, 0x19
+};
+
+/* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = {
+	0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8,
+	0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07,
+	0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17,
+	0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39,
+	0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31,
+	0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11,
+	0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7,
+	0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4
+};
+
+/* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = {
+	0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08,
+	0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b,
+	0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b,
+	0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1,
+	0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1,
+	0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f,
+	0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b,
+	0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79
+};
+
+static const struct qat_sym_hash_alg_info sha1_info = {
+	.qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE,
+	.qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE,
+	.qshai_state_size = QAT_HASH_SHA1_STATE_SIZE,
+	.qshai_init_state = sha1_initial_state,
+	.qshai_sah = &auth_hash_hmac_sha1,
+	.qshai_state_offset = 0,
+	.qshai_state_word = 4,
+};
+
+static const struct qat_sym_hash_alg_info sha256_info = {
+	.qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE,
+	.qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE,
+	.qshai_state_size = QAT_HASH_SHA256_STATE_SIZE,
+	.qshai_init_state = sha256_initial_state,
+	.qshai_sah = &auth_hash_hmac_sha2_256,
+	.qshai_state_offset = offsetof(SHA256_CTX, state),
+	.qshai_state_word = 4,
+};
+
+static const struct qat_sym_hash_alg_info sha384_info = {
+	.qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE,
+	.qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE,
+	.qshai_state_size = QAT_HASH_SHA384_STATE_SIZE,
+	.qshai_init_state = sha384_initial_state,
+	.qshai_sah = &auth_hash_hmac_sha2_384,
+	.qshai_state_offset = offsetof(SHA384_CTX, state),
+	.qshai_state_word = 8,
+};
+
+static const struct qat_sym_hash_alg_info sha512_info = {
+	.qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE,
+	.qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE,
+	.qshai_state_size = QAT_HASH_SHA512_STATE_SIZE,
+	.qshai_init_state = sha512_initial_state,
+	.qshai_sah = &auth_hash_hmac_sha2_512,
+	.qshai_state_offset = offsetof(SHA512_CTX, state),
+	.qshai_state_word = 8,
+};
+
+static const struct qat_sym_hash_alg_info aes_gcm_info = {
+	.qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE,
+	.qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE,
+	.qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE,
+	.qshai_sah = &auth_hash_nist_gmac_aes_128,
+};
+
+/* Hash QAT specific structures */
+
+static const struct qat_sym_hash_qat_info sha1_config = {
+	.qshqi_algo_enc = HW_AUTH_ALGO_SHA1,
+	.qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE,
+	.qshqi_state1_len = HW_SHA1_STATE1_SZ,
+	.qshqi_state2_len = HW_SHA1_STATE2_SZ,
+};
+
+static const struct qat_sym_hash_qat_info sha256_config = {
+	.qshqi_algo_enc = HW_AUTH_ALGO_SHA256,
+	.qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE,
+	.qshqi_state1_len = HW_SHA256_STATE1_SZ,
+	.qshqi_state2_len = HW_SHA256_STATE2_SZ
+};
+
+static const struct qat_sym_hash_qat_info sha384_config = {
+	.qshqi_algo_enc = HW_AUTH_ALGO_SHA384,
+	.qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE,
+	.qshqi_state1_len = HW_SHA384_STATE1_SZ,
+	.qshqi_state2_len = HW_SHA384_STATE2_SZ
+};
+
+static const struct qat_sym_hash_qat_info sha512_config = {
+	.qshqi_algo_enc = HW_AUTH_ALGO_SHA512,
+	.qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE,
+	.qshqi_state1_len = HW_SHA512_STATE1_SZ,
+	.qshqi_state2_len = HW_SHA512_STATE2_SZ
+};
+
+static const struct qat_sym_hash_qat_info aes_gcm_config = {
+	.qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128,
+	.qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE,
+	.qshqi_state1_len = HW_GALOIS_128_STATE1_SZ,
+	.qshqi_state2_len =
+	    HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ,
+};
+
+static const struct qat_sym_hash_def qat_sym_hash_defs[] = {
+	[QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config },
+	[QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config },
+	[QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config },
+	[QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config },
+	[QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config },
+};
+
+static const struct qat_product *qat_lookup(device_t);
+static int	qat_probe(device_t);
+static int	qat_attach(device_t);
+static int	qat_init(struct device *);
+static int	qat_start(struct device *);
+static int	qat_detach(device_t);
+
+static int	qat_newsession(device_t dev, crypto_session_t cses,
+		    const struct crypto_session_params *csp);
+static void	qat_freesession(device_t dev, crypto_session_t cses);
+
+static int	qat_setup_msix_intr(struct qat_softc *);
+
+static void	qat_etr_init(struct qat_softc *);
+static void	qat_etr_deinit(struct qat_softc *);
+static void	qat_etr_bank_init(struct qat_softc *, int);
+static void	qat_etr_bank_deinit(struct qat_softc *sc, int);
+
+static void	qat_etr_ap_bank_init(struct qat_softc *);
+static void	qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int);
+static void	qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *,
+		    uint32_t, int);
+static void	qat_etr_ap_bank_setup_ring(struct qat_softc *,
+		    struct qat_ring *);
+static int	qat_etr_verify_ring_size(uint32_t, uint32_t);
+
+static int	qat_etr_ring_intr(struct qat_softc *, struct qat_bank *,
+		    struct qat_ring *);
+static void	qat_etr_bank_intr(void *);
+
+static void	qat_arb_update(struct qat_softc *, struct qat_bank *);
+
+static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie(
+		    struct qat_crypto_bank *);
+static void	qat_crypto_free_sym_cookie(struct qat_crypto_bank *,
+		    struct qat_sym_cookie *);
+static int	qat_crypto_setup_ring(struct qat_softc *,
+		    struct qat_crypto_bank *);
+static int	qat_crypto_bank_init(struct qat_softc *,
+		    struct qat_crypto_bank *);
+static int	qat_crypto_init(struct qat_softc *);
+static void	qat_crypto_deinit(struct qat_softc *);
+static int	qat_crypto_start(struct qat_softc *);
+static void	qat_crypto_stop(struct qat_softc *);
+static int	qat_crypto_sym_rxintr(struct qat_softc *, void *, void *);
+
+static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver");
+
+static const struct qat_product *
+qat_lookup(device_t dev)
+{
+	const struct qat_product *qatp;
+
+	for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) {
+		if (pci_get_vendor(dev) == qatp->qatp_vendor &&
+		    pci_get_device(dev) == qatp->qatp_product)
+			return qatp;
+	}
+	return NULL;
+}
+
+static int
+qat_probe(device_t dev)
+{
+	const struct qat_product *prod;
+
+	prod = qat_lookup(dev);
+	if (prod != NULL) {
+		device_set_desc(dev, prod->qatp_name);
+		return BUS_PROBE_DEFAULT;
+	}
+	return ENXIO;
+}
+
+static int
+qat_attach(device_t dev)
+{
+	struct qat_softc *sc = device_get_softc(dev);
+	const struct qat_product *qatp;
+	bus_size_t msixtbl_offset;
+	int bar, count, error, i, msixoff, msixtbl_bar;
+
+	sc->sc_dev = dev;
+	sc->sc_rev = pci_get_revid(dev);
+
+	qatp = qat_lookup(dev);
+	memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw));
+
+	/* Determine active accelerators and engines */
+	sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc);
+	sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc);
+
+	sc->sc_accel_num = 0;
+	for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) {
+		if (sc->sc_accel_mask & (1 << i))
+			sc->sc_accel_num++;
+	}
+	sc->sc_ae_num = 0;
+	for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) {
+		if (sc->sc_ae_mask & (1 << i))
+			sc->sc_ae_num++;
+	}
+
+	if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) {
+		device_printf(sc->sc_dev, "couldn't find acceleration");
+		goto fail;
+	}
+
+	MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL);
+	MPASS(sc->sc_ae_num <= MAX_NUM_AE);
+
+	/* Determine SKU and capabilities */
+	sc->sc_sku = sc->sc_hw.qhw_get_sku(sc);
+	sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc);
+	sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc);
+
+	/* Map BARs */
+	msixtbl_bar = 0;
+	msixtbl_offset = 0;
+	if (pci_find_cap(dev, PCIY_MSIX, &msixoff) == 0) {
+		uint32_t msixtbl;
+		msixtbl = pci_read_config(dev, msixoff + PCIR_MSIX_TABLE, 4);
+		msixtbl_offset = msixtbl & ~PCIM_MSIX_BIR_MASK;
+		msixtbl_bar = PCIR_BAR(msixtbl & PCIM_MSIX_BIR_MASK);
+	}
+
+	i = 0;
+	if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) {
+		MPASS(sc->sc_hw.qhw_sram_bar_id == 0);
+		uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4);
+		/* Skip SRAM BAR */
+		i = (fusectl & FUSECTL_MASK) ? 1 : 0;
+	}
+	for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) {
+		uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4);
+		if (val == 0 || !PCI_BAR_MEM(val))
+			continue;
+
+		sc->sc_rid[i] = PCIR_BAR(bar);
+		sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+		    &sc->sc_rid[i], RF_ACTIVE);
+		if (sc->sc_res[i] == NULL) {
+			device_printf(dev, "couldn't map BAR %d\n", bar);
+			goto fail;
+		}
+
+		sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]);
+		sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]);
+
+		i++;
+		if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64)
+			bar++;
+	}
+
+	pci_enable_busmaster(dev);
+
+	count = sc->sc_hw.qhw_num_banks + 1;
+	if (pci_msix_count(dev) < count) {
+		device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n",
+		    pci_msix_count(dev), count);
+		goto fail;
+	}
+	error = pci_alloc_msix(dev, &count);
+	if (error != 0) {
+		device_printf(dev, "failed to allocate MSI-X vectors\n");
+		goto fail;
+	}
+
+	error = qat_init(dev);
+	if (error == 0)
+		return 0;
+
+fail:
+	qat_detach(dev);
+	return ENXIO;
+}
+
+static int
+qat_init(device_t dev)
+{
+	struct qat_softc *sc = device_get_softc(dev);
+	int error;
+
+	qat_etr_init(sc);
+
+	if (sc->sc_hw.qhw_init_admin_comms != NULL &&
+	    (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) {
+		device_printf(sc->sc_dev,
+		    "Could not initialize admin comms: %d\n", error);
+		return error;
+	}
+
+	if (sc->sc_hw.qhw_init_arb != NULL &&
+	    (error = sc->sc_hw.qhw_init_arb(sc)) != 0) {
+		device_printf(sc->sc_dev,
+		    "Could not initialize hw arbiter: %d\n", error);
+		return error;
+	}
+
+	error = qat_ae_init(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Could not initialize Acceleration Engine: %d\n", error);
+		return error;
+	}
+
+	error = qat_aefw_load(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Could not load firmware: %d\n", error);
+		return error;
+	}
+
+	error = qat_setup_msix_intr(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Could not setup interrupts: %d\n", error);
+		return error;
+	}
+
+	sc->sc_hw.qhw_enable_intr(sc);
+
+	error = qat_crypto_init(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Could not initialize service: %d\n", error);
+		return error;
+	}
+
+	if (sc->sc_hw.qhw_enable_error_correction != NULL)
+		sc->sc_hw.qhw_enable_error_correction(sc);
+
+	if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL &&
+	    (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) {
+		device_printf(sc->sc_dev,
+		    "Could not initialize watchdog timer: %d\n", error);
+		return error;
+	}
+
+	error = qat_start(dev);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Could not start: %d\n", error);
+		return error;
+	}
+
+	return 0;
+}
+
+static int
+qat_start(device_t dev)
+{
+	struct qat_softc *sc = device_get_softc(dev);
+	int error;
+
+	error = qat_ae_start(sc);
+	if (error)
+		return error;
+
+	if (sc->sc_hw.qhw_send_admin_init != NULL &&
+	    (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) {
+		return error;
+	}
+
+	error = qat_crypto_start(sc);
+	if (error)
+		return error;
+
+	return 0;
+}
+
+static int
+qat_detach(device_t dev)
+{
+	struct qat_softc *sc;
+	int bar, i;
+
+	sc = device_get_softc(dev);
+
+	qat_crypto_stop(sc);
+	qat_crypto_deinit(sc);
+	qat_aefw_unload(sc);
+
+	if (sc->sc_etr_banks != NULL) {
+		for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) {
+			struct qat_bank *qb = &sc->sc_etr_banks[i];
+
+			if (qb->qb_ih_cookie != NULL)
+				(void)bus_teardown_intr(dev, qb->qb_ih,
+				    qb->qb_ih_cookie);
+			if (qb->qb_ih != NULL)
+				(void)bus_release_resource(dev, SYS_RES_IRQ,
+				    i + 1, qb->qb_ih);
+		}
+	}
+	if (sc->sc_ih_cookie != NULL) {
+		(void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie);
+		sc->sc_ih_cookie = NULL;
+	}
+	if (sc->sc_ih != NULL) {
+		(void)bus_release_resource(dev, SYS_RES_IRQ, i + 1, sc->sc_ih);
+		sc->sc_ih = NULL;
+	}
+	pci_release_msi(dev);
+
+	qat_etr_deinit(sc);
+
+	for (bar = 0; bar < MAX_BARS; bar++) {
+		if (sc->sc_res[bar] != NULL) {
+			(void)bus_release_resource(dev, SYS_RES_MEMORY,
+			    sc->sc_rid[bar], sc->sc_res[bar]);
+			sc->sc_res[bar] = NULL;
+		}
+	}
+
+	return 0;
+}
+
+void *
+qat_alloc_mem(size_t size)
+{
+	return (malloc(size, M_QAT, M_WAITOK | M_ZERO));
+}
+
+void
+qat_free_mem(void *ptr)
+{
+	free(ptr, M_QAT);
+}
+
+static void
+qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg,
+    int error)
+{
+	struct qat_dmamem *qdm;
+
+	if (error != 0)
+		return;
+
+	KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg));
+	qdm = arg;
+	qdm->qdm_dma_seg = segs[0];
+}
+
+int
+qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm,
+    int nseg, bus_size_t size, bus_size_t alignment)
+{
+	int error;
+
+	KASSERT(qdm->qdm_dma_vaddr == NULL,
+	    ("%s: DMA memory descriptor in use", __func__));
+
+	error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
+	    alignment, 0, 		/* alignment, boundary */
+	    BUS_SPACE_MAXADDR,		/* lowaddr */
+	    BUS_SPACE_MAXADDR, 		/* highaddr */
+	    NULL, NULL, 		/* filter, filterarg */
+	    size,			/* maxsize */
+	    nseg,			/* nsegments */
+	    size,			/* maxsegsize */
+	    BUS_DMA_COHERENT,		/* flags */
+	    NULL, NULL,			/* lockfunc, lockarg */
+	    &qdm->qdm_dma_tag);
+	if (error != 0)
+		return error;
+
+	error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr,
+	    BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+	    &qdm->qdm_dma_map);
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "couldn't allocate dmamem, error = %d\n", error);
+		goto fail_0;
+	}
+
+	error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map,
+	    qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm,
+	    BUS_DMA_NOWAIT);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "couldn't load dmamem map, error = %d\n", error);
+		goto fail_1;
+	}
+
+	return 0;
+fail_1:
+	bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map);
+fail_0:
+	bus_dma_tag_destroy(qdm->qdm_dma_tag);
+	return error;
+}
+
+void
+qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm)
+{
+	if (qdm->qdm_dma_tag != NULL) {
+		bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map);
+		bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr,
+		    qdm->qdm_dma_map);
+		bus_dma_tag_destroy(qdm->qdm_dma_tag);
+		explicit_bzero(qdm, sizeof(*qdm));
+	}
+}
+
+static int
+qat_setup_msix_intr(struct qat_softc *sc)
+{
+	device_t dev;
+	int error, i, rid;
+
+	dev = sc->sc_dev;
+
+	for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) {
+		struct qat_bank *qb = &sc->sc_etr_banks[i - 1];
+
+		rid = i;
+		qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+		    RF_ACTIVE);
+		if (qb->qb_ih == NULL) {
+			device_printf(dev,
+			    "failed to allocate bank intr resource\n");
+			return ENXIO;
+		}
+		error = bus_setup_intr(dev, qb->qb_ih,
+		    INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb,
+		    &qb->qb_ih_cookie);
+		if (error != 0) {
+			device_printf(dev, "failed to set up bank intr\n");
+			return error;
+		}
+		error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus);
+		if (error != 0)
+			device_printf(dev, "failed to bind intr %d\n", i);
+	}
+
+	rid = i;
+	sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+	    RF_ACTIVE);
+	if (sc->sc_ih == NULL)
+		return ENXIO;
+	error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE,
+	    NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie);
+
+	return error;
+}
+
+static void
+qat_etr_init(struct qat_softc *sc)
+{
+	int i;
+
+	sc->sc_etr_banks = qat_alloc_mem(
+	    sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks);
+
+	for (i = 0; i < sc->sc_hw.qhw_num_banks; i++)
+		qat_etr_bank_init(sc, i);
+
+	if (sc->sc_hw.qhw_num_ap_banks) {
+		sc->sc_etr_ap_banks = qat_alloc_mem(
+		    sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks);
+		qat_etr_ap_bank_init(sc);
+	}
+}
+
+static void
+qat_etr_deinit(struct qat_softc *sc)
+{
+	int i;
+
+	if (sc->sc_etr_banks != NULL) {
+		for (i = 0; i < sc->sc_hw.qhw_num_banks; i++)
+			qat_etr_bank_deinit(sc, i);
+		qat_free_mem(sc->sc_etr_banks);
+		sc->sc_etr_banks = NULL;
+	}
+	if (sc->sc_etr_ap_banks != NULL) {
+		qat_free_mem(sc->sc_etr_ap_banks);
+		sc->sc_etr_ap_banks = NULL;
+	}
+}
+
+static void
+qat_etr_bank_init(struct qat_softc *sc, int bank)
+{
+	struct qat_bank *qb = &sc->sc_etr_banks[bank];
+	int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap;
+
+	MPASS(bank < sc->sc_hw.qhw_num_banks);
+
+	mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF);
+
+	qb->qb_sc = sc;
+	qb->qb_bank = bank;
+	qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT;
+
+	/* Clean CSRs for all rings within the bank */
+	for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) {
+		struct qat_ring *qr = &qb->qb_et_rings[i];
+
+		qat_etr_bank_ring_write_4(sc, bank, i,
+		    ETR_RING_CONFIG, 0);
+		qat_etr_bank_ring_base_write_8(sc, bank, i, 0);
+
+		if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) {
+			qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t));
+		} else if (sc->sc_hw.qhw_tx_rings_mask &
+		    (1 << (i - tx_rx_gap))) {
+			/* Share inflight counter with rx and tx */
+			qr->qr_inflight =
+			    qb->qb_et_rings[i - tx_rx_gap].qr_inflight;
+		}
+	}
+
+	if (sc->sc_hw.qhw_init_etr_intr != NULL) {
+		sc->sc_hw.qhw_init_etr_intr(sc, bank);
+	} else {
+		/* common code in qat 1.7 */
+		qat_etr_bank_write_4(sc, bank, ETR_INT_REG,
+		    ETR_INT_REG_CLEAR_MASK);
+		for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank /
+		    ETR_RINGS_PER_INT_SRCSEL; i++) {
+			qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL +
+			    (i * ETR_INT_SRCSEL_NEXT_OFFSET),
+			    ETR_INT_SRCSEL_MASK);
+		}
+	}
+}
+
+static void
+qat_etr_bank_deinit(struct qat_softc *sc, int bank)
+{
+	struct qat_bank *qb;
+	struct qat_ring *qr;
+	int i;
+
+	qb = &sc->sc_etr_banks[bank];
+	for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) {
+		if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) {
+			qr = &qb->qb_et_rings[i];
+			qat_free_mem(qr->qr_inflight);
+		}
+	}
+}
+
+static void
+qat_etr_ap_bank_init(struct qat_softc *sc)
+{
+	int ap_bank;
+
+	for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) {
+		struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank];
+
+		qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK,
+		    ETR_AP_NF_MASK_INIT);
+		qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0);
+		qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK,
+		    ETR_AP_NE_MASK_INIT);
+		qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0);
+
+		memset(qab, 0, sizeof(*qab));
+	}
+}
+
+static void
+qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask)
+{
+	if (set_mask)
+		*ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring));
+	else
+		*ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring));
+}
+
+static void
+qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest,
+    uint32_t ring, int set_dest)
+{
+	uint32_t ae_mask;
+	uint8_t mailbox, ae, nae;
+	uint8_t *dest = (uint8_t *)ap_dest;
+
+	mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring);
+
+	nae = 0;
+	ae_mask = sc->sc_ae_mask;
+	for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) {
+		if ((ae_mask & (1 << ae)) == 0)
+			continue;
+
+		if (set_dest) {
+			dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) |
+			    __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) |
+			    ETR_AP_DEST_ENABLE;
+		} else {
+			dest[nae] = 0;
+		}
+		nae++;
+		if (nae == ETR_MAX_AE_PER_MAILBOX)
+			break;
+	}
+}
+
+static void
+qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr)
+{
+	struct qat_ap_bank *qab;
+	int ap_bank;
+
+	if (sc->sc_hw.qhw_num_ap_banks == 0)
+		return;
+
+	ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring);
+	MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks);
+	qab = &sc->sc_etr_ap_banks[ap_bank];
+
+	if (qr->qr_cb == NULL) {
+		qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1);
+		if (!qab->qab_ne_dest) {
+			qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest,
+			    qr->qr_ring, 1);
+			qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST,
+			    qab->qab_ne_dest);
+		}
+	} else {
+		qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1);
+		if (!qab->qab_nf_dest) {
+			qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest,
+			    qr->qr_ring, 1);
+			qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST,
+			    qab->qab_nf_dest);
+		}
+	}
+}
+
+static int
+qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs)
+{
+	int i = QAT_MIN_RING_SIZE;
+
+	for (; i <= QAT_MAX_RING_SIZE; i++)
+		if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i))
+			return i;
+
+	return QAT_DEFAULT_RING_SIZE;
+}
+
+int
+qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring,
+    uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg,
+    const char *name, struct qat_ring **rqr)
+{
+	struct qat_bank *qb;
+	struct qat_ring *qr = NULL;
+	int error;
+	uint32_t ring_size_bytes, ring_config;
+	uint64_t ring_base;
+	uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512;
+	uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0;
+
+	MPASS(bank < sc->sc_hw.qhw_num_banks);
+
+	/* Allocate a ring from specified bank */
+	qb = &sc->sc_etr_banks[bank];
+
+	if (ring >= sc->sc_hw.qhw_num_rings_per_bank)
+		return EINVAL;
+	if (qb->qb_allocated_rings & (1 << ring))
+		return ENOENT;
+	qr = &qb->qb_et_rings[ring];
+	qb->qb_allocated_rings |= 1 << ring;
+
+	/* Initialize allocated ring */
+	qr->qr_ring = ring;
+	qr->qr_bank = bank;
+	qr->qr_name = name;
+	qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring;
+	qr->qr_ring_mask = (1 << ring);
+	qr->qr_cb = cb;
+	qr->qr_cb_arg = cb_arg;
+
+	/* Setup the shadow variables */
+	qr->qr_head = 0;
+	qr->qr_tail = 0;
+	qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size);
+	qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs);
+
+	/*
+	 * To make sure that ring is alligned to ring size allocate
+	 * at least 4k and then tell the user it is smaller.
+	 */
+	ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size);
+	ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes);
+	error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes,
+	    ring_size_bytes);
+	if (error)
+		return error;
+
+	qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr;
+	qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr;
+
+	memset(qr->qr_ring_vaddr, QAT_RING_PATTERN,
+	    qr->qr_dma.qdm_dma_seg.ds_len);
+
+	bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
+	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+
+	if (cb == NULL) {
+		ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size);
+	} else {
+		ring_config =
+		    ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne);
+	}
+	qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config);
+
+	ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size);
+	qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base);
+
+	if (sc->sc_hw.qhw_init_arb != NULL)
+		qat_arb_update(sc, qb);
+
+	mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF);
+
+	qat_etr_ap_bank_setup_ring(sc, qr);
+
+	if (cb != NULL) {
+		uint32_t intr_mask;
+
+		qb->qb_intr_mask |= qr->qr_ring_mask;
+		intr_mask = qb->qb_intr_mask;
+
+		qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask);
+		qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL,
+		    ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time);
+	}
+
+	*rqr = qr;
+
+	return 0;
+}
+
+static inline u_int
+qat_modulo(u_int data, u_int shift)
+{
+	u_int div = data >> shift;
+	u_int mult = div << shift;
+	return data - mult;
+}
+
+int
+qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg)
+{
+	uint32_t inflight;
+	uint32_t *addr;
+
+	mtx_lock(&qr->qr_ring_mtx);
+
+	inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1;
+	if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) {
+		atomic_subtract_32(qr->qr_inflight, 1);
+		qr->qr_need_wakeup = true;
+		mtx_unlock(&qr->qr_ring_mtx);
+		counter_u64_add(sc->sc_ring_full_restarts, 1);
+		return ERESTART;
+	}
+
+	addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail);
+
+	memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size));
+
+	bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
+	    BUS_DMASYNC_PREWRITE);
+
+	qr->qr_tail = qat_modulo(qr->qr_tail +
+	    QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size),
+	    QAT_RING_SIZE_MODULO(qr->qr_ring_size));
+
+	qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring,
+	    ETR_RING_TAIL_OFFSET, qr->qr_tail);
+
+	mtx_unlock(&qr->qr_ring_mtx);
+
+	return 0;
+}
+
+static int
+qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb,
+    struct qat_ring *qr)
+{
+	uint32_t *msg, nmsg = 0;
+	int handled = 0;
+	bool blocked = false;
+
+	mtx_lock(&qr->qr_ring_mtx);
+
+	msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head);
+
+	bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
+	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+
+	while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) {
+		atomic_subtract_32(qr->qr_inflight, 1);
+
+		if (qr->qr_cb != NULL) {
+			mtx_unlock(&qr->qr_ring_mtx);
+			handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg);
+			mtx_lock(&qr->qr_ring_mtx);
+		}
+
+		atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG);
+
+		qr->qr_head = qat_modulo(qr->qr_head +
+		    QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size),
+		    QAT_RING_SIZE_MODULO(qr->qr_ring_size));
+		nmsg++;
+
+		msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head);
+	}
+
+	bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
+	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+
+	if (nmsg > 0) {
+		qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring,
+		    ETR_RING_HEAD_OFFSET, qr->qr_head);
+		if (qr->qr_need_wakeup) {
+			blocked = true;
+			qr->qr_need_wakeup = false;
+		}
+	}
+
+	mtx_unlock(&qr->qr_ring_mtx);
+
+	if (blocked)
+		crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ);
+
+	return handled;
+}
+
+static void
+qat_etr_bank_intr(void *arg)
+{
+	struct qat_bank *qb = arg;
+	struct qat_softc *sc = qb->qb_sc;
+	uint32_t estat;
+	int i, handled = 0;
+
+	mtx_lock(&qb->qb_bank_mtx);
+
+	qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0);
+
+	/* Now handle all the responses */
+	estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT);
+	estat &= qb->qb_intr_mask;
+
+	qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL,
+	    ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time);
+
+	mtx_unlock(&qb->qb_bank_mtx);
+
+	while ((i = ffs(estat)) != 0) {
+		struct qat_ring *qr = &qb->qb_et_rings[--i];
+		estat &= ~(1 << i);
+		handled |= qat_etr_ring_intr(sc, qb, qr);
+	}
+}
+
+void
+qat_arb_update(struct qat_softc *sc, struct qat_bank *qb)
+{
+
+	qat_arb_ringsrvarben_write_4(sc, qb->qb_bank,
+	    qb->qb_allocated_rings & 0xff);
+}
+
+static struct qat_sym_cookie *
+qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb)
+{
+	struct qat_sym_cookie *qsc;
+
+	mtx_lock(&qcb->qcb_bank_mtx);
+
+	if (qcb->qcb_symck_free_count == 0) {
+		mtx_unlock(&qcb->qcb_bank_mtx);
+		return NULL;
+	}
+
+	qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count];
+
+	mtx_unlock(&qcb->qcb_bank_mtx);
+
+	return qsc;
+}
+
+static void
+qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb,
+    struct qat_sym_cookie *qsc)
+{
+
+	explicit_bzero(qsc->qsc_iv_buf, sizeof(qsc->qsc_iv_buf));
+	explicit_bzero(qsc->qsc_auth_res, sizeof(qsc->qsc_auth_res));
+
+	mtx_lock(&qcb->qcb_bank_mtx);
+	qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc;
+	mtx_unlock(&qcb->qcb_bank_mtx);
+}
+
+void
+qat_memcpy_htobe64(void *dst, const void *src, size_t len)
+{
+	uint64_t *dst0 = dst;
+	const uint64_t *src0 = src;
+	size_t i;
+
+	MPASS(len % sizeof(*dst0) == 0);
+
+	for (i = 0; i < len / sizeof(*dst0); i++)
+		*(dst0 + i) = htobe64(*(src0 + i));
+}
+
+void
+qat_memcpy_htobe32(void *dst, const void *src, size_t len)
+{
+	uint32_t *dst0 = dst;
+	const uint32_t *src0 = src;
+	size_t i;
+
+	MPASS(len % sizeof(*dst0) == 0);
+
+	for (i = 0; i < len / sizeof(*dst0); i++)
+		*(dst0 + i) = htobe32(*(src0 + i));
+}
+
+void
+qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte)
+{
+	switch (wordbyte) {
+	case 4:
+		qat_memcpy_htobe32(dst, src, len);
+		break;
+	case 8:
+		qat_memcpy_htobe64(dst, src, len);
+		break;
+	default:
+		panic("invalid word size %u", wordbyte);
+	}
+}
+
+void
+qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc,
+    const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def,
+    uint8_t *state)
+{
+	uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
+	char zeros[AES_BLOCK_LEN];
+	int rounds;
+
+	memset(zeros, 0, sizeof(zeros));
+	rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY);
+	rijndaelEncrypt(ks, rounds, zeros, state);
+	explicit_bzero(ks, sizeof(ks));
+}
+
+void
+qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc,
+    const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def,
+    uint8_t *state1, uint8_t *state2)
+{
+	union authctx ctx;
+	const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah;
+	uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset;
+	uint32_t state_size = hash_def->qshd_alg->qshai_state_size;
+	uint32_t state_word = hash_def->qshd_alg->qshai_state_word;
+
+	hmac_init_ipad(sah, key, klen, &ctx);
+	qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size,
+	    state_word);
+	hmac_init_opad(sah, key, klen, &ctx);
+	qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size,
+	    state_word);
+	explicit_bzero(&ctx, sizeof(ctx));
+}
+
+static enum hw_cipher_algo
+qat_aes_cipher_algo(int klen)
+{
+	switch (klen) {
+	case HW_AES_128_KEY_SZ:
+		return HW_CIPHER_ALGO_AES128;
+	case HW_AES_192_KEY_SZ:
+		return HW_CIPHER_ALGO_AES192;
+	case HW_AES_256_KEY_SZ:
+		return HW_CIPHER_ALGO_AES256;
+	default:
+		panic("invalid key length %d", klen);
+	}
+}
+
+uint16_t
+qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs)
+{
+	enum hw_cipher_algo algo;
+	enum hw_cipher_dir dir;
+	enum hw_cipher_convert key_convert;
+	enum hw_cipher_mode mode;
+
+	dir = desc->qcd_cipher_dir;
+	key_convert = HW_CIPHER_NO_CONVERT;
+	mode = qs->qs_cipher_mode;
+	switch (mode) {
+	case HW_CIPHER_CBC_MODE:
+	case HW_CIPHER_XTS_MODE:
+		algo = qs->qs_cipher_algo;
+
+		/*
+		 * AES decrypt key needs to be reversed.
+		 * Instead of reversing the key at session registration,
+		 * it is instead reversed on-the-fly by setting the KEY_CONVERT
+		 * bit here.
+		 */
+		if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT)
+			key_convert = HW_CIPHER_KEY_CONVERT;
+		break;
+	case HW_CIPHER_CTR_MODE:
+		algo = qs->qs_cipher_algo;
+		dir = HW_CIPHER_ENCRYPT;
+		break;
+	default:
+		panic("unhandled cipher mode %d", mode);
+		break;
+	}
+
+	return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir);
+}
+
+uint16_t
+qat_crypto_load_auth_session(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, const struct qat_sym_hash_def **hash_def)
+{
+	enum qat_sym_hash_algorithm algo;
+
+	switch (qs->qs_auth_algo) {
+	case HW_AUTH_ALGO_SHA1:
+		algo = QAT_SYM_HASH_SHA1;
+		break;
+	case HW_AUTH_ALGO_SHA256:
+		algo = QAT_SYM_HASH_SHA256;
+		break;
+	case HW_AUTH_ALGO_SHA384:
+		algo = QAT_SYM_HASH_SHA384;
+		break;
+	case HW_AUTH_ALGO_SHA512:
+		algo = QAT_SYM_HASH_SHA512;
+		break;
+	case HW_AUTH_ALGO_GALOIS_128:
+		algo = QAT_SYM_HASH_AES_GCM;
+		break;
+	default:
+		panic("unhandled auth algorithm %d", qs->qs_auth_algo);
+		break;
+	}
+	*hash_def = &qat_sym_hash_defs[algo];
+
+	return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode,
+	    (*hash_def)->qshd_qat->qshqi_algo_enc,
+	    (*hash_def)->qshd_alg->qshai_digest_len);
+}
+
+struct qat_crypto_load_cb_arg {
+	struct qat_session	*qs;
+	struct qat_sym_cookie	*qsc;
+	struct cryptop		*crp;
+	int			error;
+};
+
+static void
+qat_crypto_load_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
+    int error)
+{
+	struct cryptop *crp;
+	struct flat_buffer_desc *flatbuf;
+	struct qat_crypto_load_cb_arg *arg;
+	struct qat_session *qs;
+	struct qat_sym_cookie *qsc;
+	bus_addr_t addr;
+	bus_size_t len;
+	int iseg, oseg, skip;
+
+	arg = _arg;
+	if (error != 0) {
+		arg->error = error;
+		return;
+	}
+
+	crp = arg->crp;
+	qs = arg->qs;
+	qsc = arg->qsc;
+
+	if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
+		/*
+		 * The firmware expects AAD to be in a contiguous buffer and
+		 * padded to a multiple of 16 bytes.  To satisfy these
+		 * constraints we bounce the AAD into a per-request buffer.
+		 */
+		crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length,
+		    qsc->qsc_gcm_aad);
+		memset(qsc->qsc_gcm_aad + crp->crp_aad_length, 0,
+		    roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) -
+		    crp->crp_aad_length);
+		skip = crp->crp_payload_start;
+	} else if (crp->crp_aad_length > 0) {
+		skip = crp->crp_aad_start;
+	} else {
+		skip = crp->crp_payload_start;
+	}
+
+	for (iseg = oseg = 0; iseg < nseg; iseg++) {
+		addr = segs[iseg].ds_addr;
+		len = segs[iseg].ds_len;
+
+		if (skip > 0) {
+			if (skip < len) {
+				addr += skip;
+				len -= skip;
+				skip = 0;
+			} else {
+				skip -= len;
+				continue;
+			}
+		}
+
+		flatbuf = &qsc->qsc_flat_bufs[oseg++];
+		flatbuf->data_len_in_bytes = (uint32_t)len;
+		flatbuf->phy_buffer = (uint64_t)addr;
+	}
+	qsc->qsc_buf_list.num_buffers = oseg;
+}
+
+static int
+qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc,
+    struct qat_crypto_desc const *desc, struct cryptop *crp)
+{
+	struct qat_crypto_load_cb_arg arg;
+	int error;
+
+	crypto_read_iv(crp, qsc->qsc_iv_buf);
+
+	arg.crp = crp;
+	arg.qs = qs;
+	arg.qsc = qsc;
+	arg.error = 0;
+	error = bus_dmamap_load_crp(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap,
+	    crp, qat_crypto_load_cb, &arg, BUS_DMA_NOWAIT);
+	if (error == 0)
+		error = arg.error;
+	return error;
+}
+
+static inline struct qat_crypto_bank *
+qat_crypto_select_bank(struct qat_crypto *qcy)
+{
+	u_int cpuid = PCPU_GET(cpuid);
+
+	return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks];
+}
+
+static int
+qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb)
+{
+	int error, i, bank;
+	int curname = 0;
+	char *name;
+
+	bank = qcb->qcb_bank;
+
+	name = qcb->qcb_ring_names[curname++];
+	snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank);
+	error = qat_etr_setup_ring(sc, qcb->qcb_bank,
+	    sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size,
+	    NULL, NULL, name, &qcb->qcb_sym_tx);
+	if (error)
+		return error;
+
+	name = qcb->qcb_ring_names[curname++];
+	snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank);
+	error = qat_etr_setup_ring(sc, qcb->qcb_bank,
+	    sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size,
+	    qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx);
+	if (error)
+		return error;
+
+	for (i = 0; i < QAT_NSYMCOOKIE; i++) {
+		struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i];
+		struct qat_sym_cookie *qsc;
+
+		error = qat_alloc_dmamem(sc, qdm, 1,
+		    sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN);
+		if (error)
+			return error;
+
+		qsc = qdm->qdm_dma_vaddr;
+		qsc->qsc_self_dmamap = qdm->qdm_dma_map;
+		qsc->qsc_self_dma_tag = qdm->qdm_dma_tag;
+		qsc->qsc_bulk_req_params_buf_paddr =
+		    qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+		    u.qsc_bulk_cookie.qsbc_req_params_buf);
+		qsc->qsc_buffer_list_desc_paddr =
+		    qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+		    qsc_buf_list);
+		qsc->qsc_iv_buf_paddr =
+		    qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+		    qsc_iv_buf);
+		qsc->qsc_auth_res_paddr =
+		    qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+		    qsc_auth_res);
+		qsc->qsc_gcm_aad_paddr =
+		    qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+		    qsc_gcm_aad);
+		qsc->qsc_content_desc_paddr =
+		    qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
+		    qsc_content_desc);
+		qcb->qcb_symck_free[i] = qsc;
+		qcb->qcb_symck_free_count++;
+
+		error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
+		    1, 0, 			/* alignment, boundary */
+		    BUS_SPACE_MAXADDR,		/* lowaddr */
+		    BUS_SPACE_MAXADDR, 		/* highaddr */
+		    NULL, NULL, 		/* filter, filterarg */
+		    QAT_MAXLEN,			/* maxsize */
+		    QAT_MAXSEG,			/* nsegments */
+		    QAT_MAXLEN,			/* maxsegsize */
+		    BUS_DMA_COHERENT,		/* flags */
+		    NULL, NULL,			/* lockfunc, lockarg */
+		    &qsc->qsc_buf_dma_tag);
+		if (error != 0)
+			return error;
+
+		error = bus_dmamap_create(qsc->qsc_buf_dma_tag,
+		    BUS_DMA_COHERENT, &qsc->qsc_buf_dmamap);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
+static int
+qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb)
+{
+	mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF);
+
+	return qat_crypto_setup_ring(sc, qcb);
+}
+
+static void
+qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb)
+{
+	struct qat_dmamem *qdm;
+	int i;
+
+	for (i = 0; i < QAT_NSYMCOOKIE; i++) {
+		qdm = &qcb->qcb_symck_dmamems[i];
+		qat_free_dmamem(sc, qdm);
+	}
+	qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma);
+	qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma);
+
+	mtx_destroy(&qcb->qcb_bank_mtx);
+}
+
+static int
+qat_crypto_init(struct qat_softc *sc)
+{
+	struct qat_crypto *qcy = &sc->sc_crypto;
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *oid;
+	struct sysctl_oid_list *children;
+	int bank, error, num_banks;
+
+	qcy->qcy_sc = sc;
+
+	if (sc->sc_hw.qhw_init_arb != NULL)
+		num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks);
+	else
+		num_banks = sc->sc_ae_num;
+
+	qcy->qcy_num_banks = num_banks;
+
+	qcy->qcy_banks =
+	    qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks);
+
+	for (bank = 0; bank < num_banks; bank++) {
+		struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank];
+		qcb->qcb_bank = bank;
+		error = qat_crypto_bank_init(sc, qcb);
+		if (error)
+			return error;
+	}
+
+	mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF);
+
+	ctx = device_get_sysctl_ctx(sc->sc_dev);
+	oid = device_get_sysctl_tree(sc->sc_dev);
+	children = SYSCTL_CHILDREN(oid);
+	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
+	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
+	children = SYSCTL_CHILDREN(oid);
+
+	sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK);
+	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts",
+	    CTLFLAG_RD, &sc->sc_gcm_aad_restarts,
+	    "GCM requests deferred due to AAD size change");
+	sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK);
+	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates",
+	    CTLFLAG_RD, &sc->sc_gcm_aad_updates,
+	    "GCM requests that required session state update");
+	sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK);
+	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full",
+	    CTLFLAG_RD, &sc->sc_ring_full_restarts,
+	    "Requests deferred due to in-flight max reached");
+
+	return 0;
+}
+
+static void
+qat_crypto_deinit(struct qat_softc *sc)
+{
+	struct qat_crypto *qcy = &sc->sc_crypto;
+	struct qat_crypto_bank *qcb;
+	int bank;
+
+	if (qcy->qcy_banks != NULL) {
+		for (bank = 0; bank < qcy->qcy_num_banks; bank++) {
+			qcb = &qcy->qcy_banks[bank];
+			qat_crypto_bank_deinit(sc, qcb);
+		}
+		qat_free_mem(qcy->qcy_banks);
+		mtx_destroy(&qcy->qcy_crypto_mtx);
+	}
+}
+
+static int
+qat_crypto_start(struct qat_softc *sc)
+{
+	struct qat_crypto *qcy;
+
+	qcy = &sc->sc_crypto;
+	qcy->qcy_cid = crypto_get_driverid(sc->sc_dev,
+	    sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE);
+	if (qcy->qcy_cid < 0) {
+		device_printf(sc->sc_dev,
+		    "could not get opencrypto driver id\n");
+		return ENOENT;
+	}
+
+	return 0;
+}
+
+static void
+qat_crypto_stop(struct qat_softc *sc)
+{
+	struct qat_crypto *qcy;
+
+	qcy = &sc->sc_crypto;
+	if (qcy->qcy_cid >= 0)
+		(void)crypto_unregister_all(qcy->qcy_cid);
+}
+
+static int
+qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg)
+{
+	char icv[QAT_SYM_HASH_BUFFER_LEN];
+	struct qat_crypto_bank *qcb = arg;
+	struct qat_crypto *qcy;
+	struct qat_session *qs;
+	struct qat_sym_cookie *qsc;
+	struct qat_sym_bulk_cookie *qsbc;
+	struct cryptop *crp;
+	int error;
+	uint16_t auth_sz;
+	bool blocked;
+
+	qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset);
+
+	qsbc = &qsc->u.qsc_bulk_cookie;
+	qcy = qsbc->qsbc_crypto;
+	qs = qsbc->qsbc_session;
+	crp = qsbc->qsbc_cb_tag;
+
+	bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
+	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+	bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap,
+	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+	bus_dmamap_unload(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap);
+
+	error = 0;
+	if ((auth_sz = qs->qs_auth_mlen) != 0) {
+		if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
+			crypto_copydata(crp, crp->crp_digest_start,
+			    auth_sz, icv);
+			if (timingsafe_bcmp(icv, qsc->qsc_auth_res,
+			    auth_sz) != 0) {
+				error = EBADMSG;
+			}
+		} else {
+			crypto_copyback(crp, crp->crp_digest_start,
+			    auth_sz, qsc->qsc_auth_res);
+		}
+	}
+
+	qat_crypto_free_sym_cookie(qcb, qsc);
+
+	blocked = false;
+	mtx_lock(&qs->qs_session_mtx);
+	MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE);
+	qs->qs_inflight--;
+	if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) {
+		blocked = true;
+		qs->qs_need_wakeup = false;
+	}
+	mtx_unlock(&qs->qs_session_mtx);
+
+	crp->crp_etype = error;
+	crypto_done(crp);
+
+	if (blocked)
+		crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ);
+
+	return 1;
+}
+
+static int
+qat_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
+	    qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) {
+		/*
+		 * AES-XTS is not supported by the NanoQAT.
+		 */
+		return EINVAL;
+	}
+
+	switch (csp->csp_mode) {
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_CBC:
+		case CRYPTO_AES_ICM:
+			if (csp->csp_ivlen != AES_BLOCK_LEN)
+				return EINVAL;
+			break;
+		case CRYPTO_AES_XTS:
+			if (csp->csp_ivlen != AES_XTS_IV_LEN)
+				return EINVAL;
+			break;
+		default:
+			return EINVAL;
+		}
+		break;
+	case CSP_MODE_DIGEST:
+		switch (csp->csp_auth_alg) {
+		case CRYPTO_SHA1:
+		case CRYPTO_SHA1_HMAC:
+		case CRYPTO_SHA2_256:
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512:
+		case CRYPTO_SHA2_512_HMAC:
+			break;
+		case CRYPTO_AES_NIST_GMAC:
+			if (csp->csp_ivlen != AES_GCM_IV_LEN)
+				return EINVAL;
+			break;
+		default:
+			return EINVAL;
+		}
+		break;
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+			if (csp->csp_ivlen != AES_GCM_IV_LEN)
+				return EINVAL;
+			break;
+		default:
+			return EINVAL;
+		}
+		break;
+	case CSP_MODE_ETA:
+		switch (csp->csp_auth_alg) {
+		case CRYPTO_SHA1_HMAC:
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+			switch (csp->csp_cipher_alg) {
+			case CRYPTO_AES_CBC:
+			case CRYPTO_AES_ICM:
+				if (csp->csp_ivlen != AES_BLOCK_LEN)
+					return EINVAL;
+				break;
+			case CRYPTO_AES_XTS:
+				if (csp->csp_ivlen != AES_XTS_IV_LEN)
+					return EINVAL;
+				break;
+			default:
+				return EINVAL;
+			}
+			break;
+		default:
+			return EINVAL;
+		}
+		break;
+	default:
+		return EINVAL;
+	}
+
+	return CRYPTODEV_PROBE_HARDWARE;
+}
+
+static int
+qat_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct qat_crypto *qcy;
+	struct qat_dmamem *qdm;
+	struct qat_session *qs;
+	struct qat_softc *sc;
+	struct qat_crypto_desc *ddesc, *edesc;
+	int error, slices;
+
+	sc = device_get_softc(dev);
+	qs = crypto_get_driver_session(cses);
+	qcy = &sc->sc_crypto;
+
+	qdm = &qs->qs_desc_mem;
+	error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG,
+	    sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN);
+	if (error != 0)
+		return error;
+
+	mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF);
+	qs->qs_aad_length = -1;
+
+	qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr;
+	qs->qs_enc_desc = edesc = ddesc + 1;
+
+	ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr;
+	ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr +
+	    offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf);
+	edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr +
+	    sizeof(struct qat_crypto_desc);
+	edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr +
+	    offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf);
+
+	qs->qs_status = QAT_SESSION_STATUS_ACTIVE;
+	qs->qs_inflight = 0;
+
+	qs->qs_cipher_key = csp->csp_cipher_key;
+	qs->qs_cipher_klen = csp->csp_cipher_klen;
+	qs->qs_auth_key = csp->csp_auth_key;
+	qs->qs_auth_klen = csp->csp_auth_klen;
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
+		qs->qs_cipher_mode = HW_CIPHER_CBC_MODE;
+		break;
+	case CRYPTO_AES_ICM:
+		qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
+		qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
+		break;
+	case CRYPTO_AES_XTS:
+		qs->qs_cipher_algo =
+		    qat_aes_cipher_algo(csp->csp_cipher_klen / 2);
+		qs->qs_cipher_mode = HW_CIPHER_XTS_MODE;
+		break;
+	case CRYPTO_AES_NIST_GCM_16:
+		qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
+		qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
+		qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128;
+		qs->qs_auth_mode = HW_AUTH_MODE1;
+		break;
+	case 0:
+		break;
+	default:
+		panic("%s: unhandled cipher algorithm %d", __func__,
+		    csp->csp_cipher_alg);
+	}
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1_HMAC:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA1;
+		qs->qs_auth_mode = HW_AUTH_MODE1;
+		break;
+	case CRYPTO_SHA1:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA1;
+		qs->qs_auth_mode = HW_AUTH_MODE0;
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA256;
+		qs->qs_auth_mode = HW_AUTH_MODE1;
+		break;
+	case CRYPTO_SHA2_256:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA256;
+		qs->qs_auth_mode = HW_AUTH_MODE0;
+		break;
+	case CRYPTO_SHA2_384_HMAC:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA384;
+		qs->qs_auth_mode = HW_AUTH_MODE1;
+		break;
+	case CRYPTO_SHA2_384:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA384;
+		qs->qs_auth_mode = HW_AUTH_MODE0;
+		break;
+	case CRYPTO_SHA2_512_HMAC:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA512;
+		qs->qs_auth_mode = HW_AUTH_MODE1;
+		break;
+	case CRYPTO_SHA2_512:
+		qs->qs_auth_algo = HW_AUTH_ALGO_SHA512;
+		qs->qs_auth_mode = HW_AUTH_MODE0;
+		break;
+	case CRYPTO_AES_NIST_GMAC:
+		qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen);
+		qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
+		qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128;
+		qs->qs_auth_mode = HW_AUTH_MODE1;
+
+		qs->qs_cipher_key = qs->qs_auth_key;
+		qs->qs_cipher_klen = qs->qs_auth_klen;
+		break;
+	case 0:
+		break;
+	default:
+		panic("%s: unhandled auth algorithm %d", __func__,
+		    csp->csp_auth_alg);
+	}
+
+	slices = 0;
+	switch (csp->csp_mode) {
+	case CSP_MODE_AEAD:
+	case CSP_MODE_ETA:
+		/* auth then decrypt */
+		ddesc->qcd_slices[0] = FW_SLICE_AUTH;
+		ddesc->qcd_slices[1] = FW_SLICE_CIPHER;
+		ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
+		ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER;
+		/* encrypt then auth */
+		edesc->qcd_slices[0] = FW_SLICE_CIPHER;
+		edesc->qcd_slices[1] = FW_SLICE_AUTH;
+		edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
+		edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH;
+		slices = 2;
+		break;
+	case CSP_MODE_CIPHER:
+		/* decrypt */
+		ddesc->qcd_slices[0] = FW_SLICE_CIPHER;
+		ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
+		ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER;
+		/* encrypt */
+		edesc->qcd_slices[0] = FW_SLICE_CIPHER;
+		edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
+		edesc->qcd_cmd_id = FW_LA_CMD_CIPHER;
+		slices = 1;
+		break;
+	case CSP_MODE_DIGEST:
+		if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
+			/* auth then decrypt */
+			ddesc->qcd_slices[0] = FW_SLICE_AUTH;
+			ddesc->qcd_slices[1] = FW_SLICE_CIPHER;
+			ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
+			ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER;
+			/* encrypt then auth */
+			edesc->qcd_slices[0] = FW_SLICE_CIPHER;
+			edesc->qcd_slices[1] = FW_SLICE_AUTH;
+			edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
+			edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH;
+			slices = 2;
+		} else {
+			ddesc->qcd_slices[0] = FW_SLICE_AUTH;
+			ddesc->qcd_cmd_id = FW_LA_CMD_AUTH;
+			edesc->qcd_slices[0] = FW_SLICE_AUTH;
+			edesc->qcd_cmd_id = FW_LA_CMD_AUTH;
+			slices = 1;
+		}
+		break;
+	default:
+		panic("%s: unhandled crypto algorithm %d, %d", __func__,
+		    csp->csp_cipher_alg, csp->csp_auth_alg);
+	}
+	ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR;
+	edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR;
+
+	qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc);
+	qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc);
+
+	if (csp->csp_auth_mlen != 0)
+		qs->qs_auth_mlen = csp->csp_auth_mlen;
+	else
+		qs->qs_auth_mlen = edesc->qcd_auth_sz;
+
+	/* Compute the GMAC by specifying a null cipher payload. */
+	if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC)
+		ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH;
+
+	return 0;
+}
+
+static void
+qat_crypto_clear_desc(struct qat_crypto_desc *desc)
+{
+	explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc));
+	explicit_bzero(desc->qcd_hash_state_prefix_buf,
+	    sizeof(desc->qcd_hash_state_prefix_buf));
+	explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache));
+}
+
+static void
+qat_freesession(device_t dev, crypto_session_t cses)
+{
+	struct qat_session *qs;
+
+	qs = crypto_get_driver_session(cses);
+	KASSERT(qs->qs_inflight == 0,
+	    ("%s: session %p has requests in flight", __func__, qs));
+
+	qat_crypto_clear_desc(qs->qs_enc_desc);
+	qat_crypto_clear_desc(qs->qs_dec_desc);
+	qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem);
+	mtx_destroy(&qs->qs_session_mtx);
+}
+
+static int
+qat_process(device_t dev, struct cryptop *crp, int hint)
+{
+	struct qat_crypto *qcy;
+	struct qat_crypto_bank *qcb;
+	struct qat_crypto_desc const *desc;
+	struct qat_session *qs;
+	struct qat_softc *sc;
+	struct qat_sym_cookie *qsc;
+	struct qat_sym_bulk_cookie *qsbc;
+	int error;
+
+	sc = device_get_softc(dev);
+	qcy = &sc->sc_crypto;
+	qs = crypto_get_driver_session(crp->crp_session);
+	qsc = NULL;
+
+	if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) {
+		error = E2BIG;
+		goto fail1;
+	}
+
+	mtx_lock(&qs->qs_session_mtx);
+	if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
+		if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) {
+			error = E2BIG;
+			mtx_unlock(&qs->qs_session_mtx);
+			goto fail1;
+		}
+
+		/*
+		 * The firmware interface for GCM annoyingly requires the AAD
+		 * size to be stored in the session's content descriptor, which
+		 * is not really meant to be updated after session
+		 * initialization.  For IPSec the AAD size is fixed so this is
+		 * not much of a problem in practice, but we have to catch AAD
+		 * size updates here so that the device code can safely update
+		 * the session's recorded AAD size.
+		 */
+		if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) {
+			if (qs->qs_inflight == 0) {
+				if (qs->qs_aad_length != -1) {
+					counter_u64_add(sc->sc_gcm_aad_updates,
+					    1);
+				}
+				qs->qs_aad_length = crp->crp_aad_length;
+			} else {
+				qs->qs_need_wakeup = true;
+				mtx_unlock(&qs->qs_session_mtx);
+				counter_u64_add(sc->sc_gcm_aad_restarts, 1);
+				error = ERESTART;
+				goto fail1;
+			}
+		}
+	}
+	qs->qs_inflight++;
+	mtx_unlock(&qs->qs_session_mtx);
+
+	qcb = qat_crypto_select_bank(qcy);
+
+	qsc = qat_crypto_alloc_sym_cookie(qcb);
+	if (qsc == NULL) {
+		error = ENOBUFS;
+		goto fail2;
+	}
+
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+		desc = qs->qs_enc_desc;
+	else
+		desc = qs->qs_dec_desc;
+
+	error = qat_crypto_load(qs, qsc, desc, crp);
+	if (error != 0)
+		goto fail2;
+
+	qsbc = &qsc->u.qsc_bulk_cookie;
+	qsbc->qsbc_crypto = qcy;
+	qsbc->qsbc_session = qs;
+	qsbc->qsbc_cb_tag = crp;
+
+	sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp);
+
+	bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap,
+	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+	bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
+	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+
+	error = qat_etr_put_msg(sc, qcb->qcb_sym_tx,
+	    (uint32_t *)qsbc->qsbc_msg);
+	if (error)
+		goto fail2;
+
+	return 0;
+
+fail2:
+	if (qsc)
+		qat_crypto_free_sym_cookie(qcb, qsc);
+	mtx_lock(&qs->qs_session_mtx);
+	qs->qs_inflight--;
+	mtx_unlock(&qs->qs_session_mtx);
+fail1:
+	crp->crp_etype = error;
+	crypto_done(crp);
+	return 0;
+}
+
+static device_method_t qat_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		qat_probe),
+	DEVMETHOD(device_attach,	qat_attach),
+	DEVMETHOD(device_detach,	qat_detach),
+
+	/* Cryptodev interface */
+	DEVMETHOD(cryptodev_probesession, qat_probesession),
+	DEVMETHOD(cryptodev_newsession,	qat_newsession),
+	DEVMETHOD(cryptodev_freesession, qat_freesession),
+	DEVMETHOD(cryptodev_process,	qat_process),
+
+	DEVMETHOD_END
+};
+
+static devclass_t qat_devclass;
+
+static driver_t qat_driver = {
+	.name		= "qat",
+	.methods	= qat_methods,
+	.size		= sizeof(struct qat_softc),
+};
+
+DRIVER_MODULE(qat, pci, qat_driver, qat_devclass, 0, 0);
+MODULE_VERSION(qat, 1);
+MODULE_DEPEND(qat, crypto, 1, 1, 1);
+MODULE_DEPEND(qat, pci, 1, 1, 1);
diff --git a/sys/dev/qat/qat_ae.c b/sys/dev/qat/qat_ae.c
new file mode 100644
index 000000000000..83695355f73d
--- /dev/null
+++ b/sys/dev/qat/qat_ae.c
@@ -0,0 +1,3456 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/firmware.h>
+#include <sys/limits.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qatvar.h"
+#include "qat_aevar.h"
+
+static int	qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
+		    uint32_t);
+static int	qat_ae_read_4(struct qat_softc *, u_char, bus_size_t,
+		    uint32_t *);
+static int	qat_ae_write_4(struct qat_softc *, u_char, bus_size_t,
+		    uint32_t);
+static void	qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t,
+		    bus_size_t, uint32_t);
+static int	qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t,
+		    bus_size_t, uint32_t *);
+
+static u_short	qat_aereg_get_10bit_addr(enum aereg_type, u_short);
+static int	qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char,
+		    enum aereg_type, u_short, uint32_t);
+static int	qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char,
+		    enum aereg_type, u_short, uint32_t *);
+static int	qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char,
+		    enum aereg_type, u_short, uint32_t);
+static int	qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char,
+		    enum aereg_type, u_short, uint32_t);
+static int	qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char,
+		    enum aereg_type, u_short, uint32_t);
+static int	qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short,
+		    u_short *, u_char *);
+static int	qat_aereg_abs_data_write(struct qat_softc *, u_char,
+		    enum aereg_type, u_short, uint32_t);
+
+static void	qat_ae_enable_ctx(struct qat_softc *, u_char, u_int);
+static void	qat_ae_disable_ctx(struct qat_softc *, u_char, u_int);
+static void	qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char);
+static void	qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char);
+static void	qat_ae_write_lm_mode(struct qat_softc *, u_char,
+		    enum aereg_type, u_char);
+static void	qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char,
+		    u_char);
+static void	qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char);
+static int	qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int,
+		    u_int);
+
+static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char);
+static int	qat_ae_is_active(struct qat_softc *, u_char);
+static int	qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int);
+
+static int	qat_ae_clear_reset(struct qat_softc *);
+static int	qat_ae_check(struct qat_softc *);
+static int	qat_ae_reset_timestamp(struct qat_softc *);
+static void	qat_ae_clear_xfer(struct qat_softc *);
+static int	qat_ae_clear_gprs(struct qat_softc *);
+
+static void	qat_ae_get_shared_ustore_ae(u_char, u_char *);
+static u_int	qat_ae_ucode_parity64(uint64_t);
+static uint64_t	qat_ae_ucode_set_ecc(uint64_t);
+static int	qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int,
+		    const uint64_t *);
+static int	qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int,
+		    uint64_t *);
+static u_int	qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *);
+static int	qat_ae_exec_ucode(struct qat_softc *, u_char, u_char,
+		    uint64_t *, u_int, int, u_int, u_int *);
+static int	qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char,
+		    int *, uint64_t *, u_int,
+		    u_int *, u_int *, u_int *, u_int *, u_int *);
+static int	qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char,
+		    u_int, u_int, u_int, u_int, u_int);
+static int	qat_ae_get_inst_num(int);
+static int	qat_ae_batch_put_lm(struct qat_softc *, u_char,
+		    struct qat_ae_batch_init_list *, size_t);
+static int	qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int);
+
+static u_int	qat_aefw_csum(char *, int);
+static const char *qat_aefw_uof_string(struct qat_softc *, size_t);
+static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *,
+		    const char *, struct uof_chunk_hdr *);
+
+static int	qat_aefw_load_mof(struct qat_softc *);
+static void	qat_aefw_unload_mof(struct qat_softc *);
+static int	qat_aefw_load_mmp(struct qat_softc *);
+static void	qat_aefw_unload_mmp(struct qat_softc *);
+
+static int	qat_aefw_mof_find_uof0(struct qat_softc *,
+		    struct mof_uof_hdr *, struct mof_uof_chunk_hdr *,
+		    u_int, size_t, const char *,
+		    size_t *, void **);
+static int	qat_aefw_mof_find_uof(struct qat_softc *);
+static int	qat_aefw_mof_parse(struct qat_softc *);
+
+static int	qat_aefw_uof_parse_image(struct qat_softc *,
+		    struct qat_uof_image *, struct uof_chunk_hdr *uch);
+static int	qat_aefw_uof_parse_images(struct qat_softc *);
+static int	qat_aefw_uof_parse(struct qat_softc *);
+
+static int	qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t,
+		    struct qat_dmamem *);
+static int	qat_aefw_auth(struct qat_softc *, struct qat_dmamem *);
+static int	qat_aefw_suof_load(struct qat_softc *sc,
+		    struct qat_dmamem *dma);
+static int	qat_aefw_suof_parse_image(struct qat_softc *,
+		    struct qat_suof_image *, struct suof_chunk_hdr *);
+static int	qat_aefw_suof_parse(struct qat_softc *);
+static int	qat_aefw_suof_write(struct qat_softc *);
+
+static int	qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *,
+		    struct qat_uof_image *);
+static int	qat_aefw_uof_init_ae(struct qat_softc *, u_char);
+static int	qat_aefw_uof_init(struct qat_softc *);
+
+static int	qat_aefw_init_memory_one(struct qat_softc *,
+		    struct uof_init_mem *);
+static void	qat_aefw_free_lm_init(struct qat_softc *, u_char);
+static int	qat_aefw_init_ustore(struct qat_softc *);
+static int	qat_aefw_init_reg(struct qat_softc *, u_char, u_char,
+		    enum aereg_type, u_short, u_int);
+static int	qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char,
+		    struct qat_uof_image *);
+static int	qat_aefw_init_memory(struct qat_softc *);
+static int	qat_aefw_init_globals(struct qat_softc *);
+static uint64_t	qat_aefw_get_uof_inst(struct qat_softc *,
+		    struct qat_uof_page *, u_int);
+static int	qat_aefw_do_pagein(struct qat_softc *, u_char,
+		    struct qat_uof_page *);
+static int	qat_aefw_uof_write_one(struct qat_softc *,
+		    struct qat_uof_image *);
+static int	qat_aefw_uof_write(struct qat_softc *);
+
+static int
+qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
+	uint32_t value)
+{
+	int times = TIMEOUT_AE_CSR;
+
+	do {
+		qat_ae_local_write_4(sc, ae, offset, value);
+		if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
+		    LOCAL_CSR_STATUS_STATUS) == 0)
+			return 0;
+
+	} while (times--);
+
+	device_printf(sc->sc_dev,
+	    "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
+	return EFAULT;
+}
+
+static int
+qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
+	uint32_t *value)
+{
+	int times = TIMEOUT_AE_CSR;
+	uint32_t v;
+
+	do {
+		v = qat_ae_local_read_4(sc, ae, offset);
+		if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) &
+		    LOCAL_CSR_STATUS_STATUS) == 0) {
+			*value = v;
+			return 0;
+		}
+	} while (times--);
+
+	device_printf(sc->sc_dev,
+	    "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset);
+	return EFAULT;
+}
+
+static void
+qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask,
+    bus_size_t offset, uint32_t value)
+{
+	int ctx;
+	uint32_t ctxptr;
+
+	MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
+	    offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
+	    offset == CTX_STS_INDIRECT ||
+	    offset == CTX_WAKEUP_EVENTS_INDIRECT ||
+	    offset == CTX_SIG_EVENTS_INDIRECT ||
+	    offset == LM_ADDR_0_INDIRECT ||
+	    offset == LM_ADDR_1_INDIRECT ||
+	    offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
+	    offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);
+
+	qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
+	for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
+		if ((ctx_mask & (1 << ctx)) == 0)
+			continue;
+		qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
+		qat_ae_write_4(sc, ae, offset, value);
+	}
+	qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
+}
+
+static int
+qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx,
+    bus_size_t offset, uint32_t *value)
+{
+	int error;
+	uint32_t ctxptr;
+
+	MPASS(offset == CTX_FUTURE_COUNT_INDIRECT ||
+	    offset == FUTURE_COUNT_SIGNAL_INDIRECT ||
+	    offset == CTX_STS_INDIRECT ||
+	    offset == CTX_WAKEUP_EVENTS_INDIRECT ||
+	    offset == CTX_SIG_EVENTS_INDIRECT ||
+	    offset == LM_ADDR_0_INDIRECT ||
+	    offset == LM_ADDR_1_INDIRECT ||
+	    offset == INDIRECT_LM_ADDR_0_BYTE_INDEX ||
+	    offset == INDIRECT_LM_ADDR_1_BYTE_INDEX);
+
+	/* save the ctx ptr */
+	qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr);
+	if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
+	    (ctx & CSR_CTX_POINTER_CONTEXT))
+		qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx);
+
+	error = qat_ae_read_4(sc, ae, offset, value);
+
+	/* restore ctx ptr */
+	if ((ctxptr & CSR_CTX_POINTER_CONTEXT) !=
+	    (ctx & CSR_CTX_POINTER_CONTEXT))
+		qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr);
+
+	return error;
+}
+
+static u_short
+qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg)
+{
+	u_short addr;
+
+	switch (regtype) {
+	case AEREG_GPA_ABS:
+	case AEREG_GPB_ABS:
+		addr = (reg & 0x7f) | 0x80;
+		break;
+	case AEREG_GPA_REL:
+	case AEREG_GPB_REL:
+		addr = reg & 0x1f;
+		break;
+	case AEREG_SR_RD_REL:
+	case AEREG_SR_WR_REL:
+	case AEREG_SR_REL:
+		addr = 0x180 | (reg & 0x1f);
+		break;
+	case AEREG_SR_INDX:
+		addr = 0x140 | ((reg & 0x3) << 1);
+		break;
+	case AEREG_DR_RD_REL:
+	case AEREG_DR_WR_REL:
+	case AEREG_DR_REL:
+		addr = 0x1c0 | (reg & 0x1f);
+		break;
+	case AEREG_DR_INDX:
+		addr = 0x100 | ((reg & 0x3) << 1);
+		break;
+	case AEREG_NEIGH_INDX:
+		addr = 0x241 | ((reg & 0x3) << 1);
+		break;
+	case AEREG_NEIGH_REL:
+		addr = 0x280 | (reg & 0x1f);
+		break;
+	case AEREG_LMEM0:
+		addr = 0x200;
+		break;
+	case AEREG_LMEM1:
+		addr = 0x220;
+		break;
+	case AEREG_NO_DEST:
+		addr = 0x300 | (reg & 0xff);
+		break;
+	default:
+		addr = AEREG_BAD_REGADDR;
+		break;
+	}
+	return (addr);
+}
+
+static int
+qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx,
+    enum aereg_type regtype, u_short relreg, uint32_t value)
+{
+	uint16_t srchi, srclo, destaddr, data16hi, data16lo;
+	uint64_t inst[] = {
+		0x0F440000000ull,	/* immed_w1[reg, val_hi16] */
+		0x0F040000000ull,	/* immed_w0[reg, val_lo16] */
+		0x0F0000C0300ull,	/* nop */
+		0x0E000010000ull	/* ctx_arb[kill] */
+	};
+	const int ninst = nitems(inst);
+	const int imm_w1 = 0, imm_w0 = 1;
+	unsigned int ctxen;
+	uint16_t mask;
+
+	/* This logic only works for GPRs and LM index registers,
+	   not NN or XFER registers! */
+	MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
+	    regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);
+
+	if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) {
+		/* determine the context mode */
+		qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+		if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
+			/* 4-ctx mode */
+			if (ctx & 0x1)
+				return EINVAL;
+			mask = 0x1f;
+		} else {
+			/* 8-ctx mode */
+			mask = 0x0f;
+		}
+		if (relreg & ~mask)
+			return EINVAL;
+	}
+	if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
+	    AEREG_BAD_REGADDR) {
+		return EINVAL;
+	}
+
+	data16lo = 0xffff & value;
+	data16hi = 0xffff & (value >> 16);
+	srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
+		(uint16_t)(0xff & data16hi));
+	srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST,
+		(uint16_t)(0xff & data16lo));
+
+	switch (regtype) {
+	case AEREG_GPA_REL:	/* A rel source */
+		inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
+		    ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff);
+		inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
+		    ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff);
+		break;
+	default:
+		inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) |
+		    ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff);
+		inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) |
+		    ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff);
+		break;
+	}
+
+	return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL);
+}
+
+static int
+qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx,
+    enum aereg_type regtype, u_short relreg, uint32_t *value)
+{
+	uint64_t inst, savucode;
+	uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi;
+	u_int uaddr, ustore_addr;
+	int error;
+	u_short mask, regaddr;
+	u_char nae;
+
+	MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL ||
+	    regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL ||
+	    regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL ||
+	    regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1);
+
+	if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) ||
+	    (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) ||
+	    (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL))
+	{
+		/* determine the context mode */
+		qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+		if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
+			/* 4-ctx mode */
+			if (ctx & 0x1)
+				return EINVAL;
+			mask = 0x1f;
+		} else {
+			/* 8-ctx mode */
+			mask = 0x0f;
+		}
+		if (relreg & ~mask)
+			return EINVAL;
+	}
+	if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) ==
+	    AEREG_BAD_REGADDR) {
+		return EINVAL;
+	}
+
+	/* instruction -- alu[--, --, B, reg] */
+	switch (regtype) {
+	case AEREG_GPA_REL:
+		/* A rel source */
+		inst = 0xA070000000ull | (regaddr & 0x3ff);
+		break;
+	default:
+		inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10));
+		break;
+	}
+
+	/* backup shared control store bit, and force AE to
+	 * none-shared mode before executing ucode snippet */
+	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
+	if (misc & AE_MISC_CONTROL_SHARE_CS) {
+		qat_ae_get_shared_ustore_ae(ae, &nae);
+		if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae))
+			return EBUSY;
+	}
+
+	nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
+
+	/* read current context */
+	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
+	qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+	/* prevent clearing the W1C bits: the breakpoint bit,
+	ECC error bit, and Parity error bit */
+	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
+
+	/* change the context */
+	if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO))
+		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
+		    ctx & ACTIVE_CTX_STATUS_ACNO);
+	/* save a ustore location */
+	if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) {
+		/* restore AE_MISC_CONTROL csr */
+		qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
+
+		/* restore the context */
+		if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
+			qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
+			    savctx & ACTIVE_CTX_STATUS_ACNO);
+		}
+		qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
+
+		return (error);
+	}
+
+	/* turn off ustore parity */
+	qat_ae_write_4(sc, ae, CTX_ENABLES,
+	    ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE));
+
+	/* save ustore-addr csr */
+	qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
+
+	/* write the ALU instruction to ustore, enable ecs bit */
+	uaddr = 0 | USTORE_ADDRESS_ECS;
+
+	/* set the uaddress */
+	qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
+	inst = qat_ae_ucode_set_ecc(inst);
+
+	ulo = (uint32_t)(inst & 0xffffffff);
+	uhi = (uint32_t)(inst >> 32);
+
+	qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
+
+	/* this will auto increment the address */
+	qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);
+
+	/* set the uaddress */
+	qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
+
+	/* delay for at least 8 cycles */
+	qat_ae_wait_num_cycles(sc, ae, 0x8, 0);
+
+	/* read ALU output -- the instruction should have been executed
+	prior to clearing the ECS in putUwords */
+	qat_ae_read_4(sc, ae, ALU_OUT, value);
+
+	/* restore ustore-addr csr */
+	qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);
+
+	/* restore the ustore */
+	error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode);
+
+	/* restore the context */
+	if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) {
+		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
+		    savctx & ACTIVE_CTX_STATUS_ACNO);
+	}
+
+	qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
+
+	/* restore AE_MISC_CONTROL csr */
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
+
+	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
+
+	return error;
+}
+
+static int
+qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
+    enum aereg_type regtype, u_short relreg, uint32_t value)
+{
+	bus_size_t addr;
+	int error;
+	uint32_t ctxen;
+	u_short mask;
+	u_short dr_offset;
+
+	MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL ||
+	    regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL);
+
+	error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+	if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
+		if (ctx & 0x1) {
+			device_printf(sc->sc_dev,
+			    "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx);
+			return EINVAL;
+		}
+		mask = 0x1f;
+		dr_offset = 0x20;
+
+	} else {
+		mask = 0x0f;
+		dr_offset = 0x10;
+	}
+
+	if (relreg & ~mask)
+		return EINVAL;
+
+	addr = relreg + (ctx << 0x5);
+
+	switch (regtype) {
+	case AEREG_SR_REL:
+	case AEREG_SR_RD_REL:
+		qat_ae_xfer_write_4(sc, ae, addr, value);
+		break;
+	case AEREG_DR_REL:
+	case AEREG_DR_RD_REL:
+		qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value);
+		break;
+	default:
+		error = EINVAL;
+	}
+
+	return error;
+}
+
+static int
+qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx,
+    enum aereg_type regtype, u_short relreg, uint32_t value)
+{
+
+	panic("notyet");
+
+	return 0;
+}
+
+static int
+qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx,
+    enum aereg_type regtype, u_short relreg, uint32_t value)
+{
+
+	panic("notyet");
+
+	return 0;
+}
+
+static int
+qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae,
+	u_short absreg, u_short *relreg, u_char *ctx)
+{
+	uint32_t ctxen;
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+	if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
+		/* 4-ctx mode */
+		*relreg = absreg & 0x1f;
+		*ctx = (absreg >> 0x4) & 0x6;
+	} else {
+		/* 8-ctx mode */
+		*relreg = absreg & 0x0f;
+		*ctx = (absreg >> 0x4) & 0x7;
+	}
+
+	return 0;
+}
+
+static int
+qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae,
+	enum aereg_type regtype, u_short absreg, uint32_t value)
+{
+	int error;
+	u_short relreg;
+	u_char ctx;
+
+	qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx);
+
+	switch (regtype) {
+	case AEREG_GPA_ABS:
+		MPASS(absreg < MAX_GPR_REG);
+		error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL,
+		    relreg, value);
+		break;
+	case AEREG_GPB_ABS:
+		MPASS(absreg < MAX_GPR_REG);
+		error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL,
+		    relreg, value);
+		break;
+	case AEREG_DR_RD_ABS:
+		MPASS(absreg < MAX_XFER_REG);
+		error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL,
+		    relreg, value);
+		break;
+	case AEREG_SR_RD_ABS:
+		MPASS(absreg < MAX_XFER_REG);
+		error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL,
+		    relreg, value);
+		break;
+	case AEREG_DR_WR_ABS:
+		MPASS(absreg < MAX_XFER_REG);
+		error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL,
+		    relreg, value);
+		break;
+	case AEREG_SR_WR_ABS:
+		MPASS(absreg < MAX_XFER_REG);
+		error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL,
+		    relreg, value);
+		break;
+	case AEREG_NEIGH_ABS:
+		MPASS(absreg < MAX_NN_REG);
+		if (absreg >= MAX_NN_REG)
+			return EINVAL;
+		error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL,
+		    relreg, value);
+		break;
+	default:
+		panic("Invalid Register Type");
+	}
+
+	return error;
+}
+
+static void
+qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
+{
+	uint32_t ctxen;
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
+
+	if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) {
+		ctx_mask &= 0x55;
+	} else {
+		ctx_mask &= 0xff;
+	}
+
+	ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE);
+	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
+}
+
+static void
+qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask)
+{
+	uint32_t ctxen;
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
+	ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE));
+	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
+}
+
+static void
+qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode)
+{
+	uint32_t val, nval;
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
+	val &= CTX_ENABLES_IGNORE_W1C_MASK;
+
+	if (mode == 4)
+		nval = val | CTX_ENABLES_INUSE_CONTEXTS;
+	else
+		nval = val & ~CTX_ENABLES_INUSE_CONTEXTS;
+
+	if (val != nval)
+		qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
+}
+
+static void
+qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode)
+{
+	uint32_t val, nval;
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
+	val &= CTX_ENABLES_IGNORE_W1C_MASK;
+
+	if (mode)
+		nval = val | CTX_ENABLES_NN_MODE;
+	else
+		nval = val & ~CTX_ENABLES_NN_MODE;
+
+	if (val != nval)
+		qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
+}
+
+static void
+qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae,
+	enum aereg_type lm, u_char mode)
+{
+	uint32_t val, nval;
+	uint32_t bit;
+
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
+	val &= CTX_ENABLES_IGNORE_W1C_MASK;
+
+	switch (lm) {
+	case AEREG_LMEM0:
+		bit = CTX_ENABLES_LMADDR_0_GLOBAL;
+		break;
+	case AEREG_LMEM1:
+		bit = CTX_ENABLES_LMADDR_1_GLOBAL;
+		break;
+	default:
+		panic("invalid lmem reg type");
+		break;
+	}
+
+	if (mode)
+		nval = val | bit;
+	else
+		nval = val & ~bit;
+
+	if (val != nval)
+		qat_ae_write_4(sc, ae, CTX_ENABLES, nval);
+}
+
+static void
+qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode)
+{
+	uint32_t val, nval;
+
+	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
+
+	if (mode == 1)
+		nval = val | AE_MISC_CONTROL_SHARE_CS;
+	else
+		nval = val & ~AE_MISC_CONTROL_SHARE_CS;
+
+	if (val != nval)
+		qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval);
+}
+
+static void
+qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode)
+{
+	u_char nae;
+
+	qat_ae_get_shared_ustore_ae(ae, &nae);
+
+	qat_ae_write_shared_cs_mode0(sc, ae, mode);
+
+	if ((sc->sc_ae_mask & (1 << nae))) {
+		qat_ae_write_shared_cs_mode0(sc, nae, mode);
+	}
+}
+
+static int
+qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae,
+	u_int reload_size, int shared_mode, u_int ustore_dram_addr)
+{
+	uint32_t val, cs_reload;
+
+	switch (reload_size) {
+	case 0:
+		cs_reload = 0x0;
+		break;
+	case QAT_2K:
+		cs_reload = 0x1;
+		break;
+	case QAT_4K:
+		cs_reload = 0x2;
+		break;
+	case QAT_8K:
+		cs_reload = 0x3;
+		break;
+	default:
+		return EINVAL;
+	}
+
+	if (cs_reload)
+		QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr;
+
+	QAT_AE(sc, ae).qae_reload_size = reload_size;
+
+	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
+	val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD |
+	    AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS);
+	val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) |
+	    __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD);
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);
+
+	return 0;
+}
+
+static enum qat_ae_status
+qat_ae_get_status(struct qat_softc *sc, u_char ae)
+{
+	int error;
+	uint32_t val = 0;
+
+	error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
+	if (error || val & CTX_ENABLES_ENABLE)
+		return QAT_AE_ENABLED;
+
+	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
+	if (val & ACTIVE_CTX_STATUS_ABO)
+		return QAT_AE_ACTIVE;
+
+	return QAT_AE_DISABLED;
+}
+
+
+static int
+qat_ae_is_active(struct qat_softc *sc, u_char ae)
+{
+	uint32_t val;
+
+	if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
+		return 1;
+
+	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val);
+	if (val & ACTIVE_CTX_STATUS_ABO)
+		return 1;
+	else
+		return 0;
+}
+
+/* returns 1 if actually waited for specified number of cycles */
+static int
+qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check)
+{
+	uint32_t cnt, actx;
+	int pcnt, ccnt, elapsed, times;
+
+	qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
+	pcnt = cnt & 0xffff;
+
+	times = TIMEOUT_AE_CHECK;
+	do {
+		qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
+		ccnt = cnt & 0xffff;
+
+		elapsed = ccnt - pcnt;
+		if (elapsed == 0) {
+			times--;
+		}
+		if (times <= 0) {
+			device_printf(sc->sc_dev,
+			    "qat_ae_wait_num_cycles timeout\n");
+			return -1;
+		}
+
+		if (elapsed < 0)
+			elapsed += 0x10000;
+
+		if (elapsed >= CYCLES_FROM_READY2EXE && check) {
+			if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS,
+			    &actx) == 0) {
+				if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
+					return 0;
+			}
+		}
+	} while (cycles > elapsed);
+
+	if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) {
+		if ((actx & ACTIVE_CTX_STATUS_ABO) == 0)
+			return 0;
+	}
+
+	return 1;
+}
+
+int
+qat_ae_init(struct qat_softc *sc)
+{
+	int error;
+	uint32_t mask, val = 0;
+	u_char ae;
+
+	/* XXX adf_initSysMemInfo */
+
+	/* XXX Disable clock gating for some chip if debug mode */
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		struct qat_ae *qae = &sc->sc_ae[ae];
+		if (!(mask & 1))
+			continue;
+
+		qae->qae_ustore_size = USTORE_SIZE;
+
+		qae->qae_free_addr = 0;
+		qae->qae_free_size = USTORE_SIZE;
+		qae->qae_live_ctx_mask = AE_ALL_CTX;
+		qae->qae_ustore_dram_addr = 0;
+		qae->qae_reload_size = 0;
+	}
+
+	/* XXX Enable attention interrupt */
+
+	error = qat_ae_clear_reset(sc);
+	if (error)
+		return error;
+
+	qat_ae_clear_xfer(sc);
+
+	if (!sc->sc_hw.qhw_fw_auth) {
+		error = qat_ae_clear_gprs(sc);
+		if (error)
+			return error;
+	}
+
+	/* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val);
+		val |= 0x1;
+		qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val);
+	}
+
+	error = qat_ae_clear_reset(sc);
+	if (error)
+		return error;
+
+	/* XXX XXX XXX Clean MMP memory if mem scrub is supported */
+	/* halMem_ScrubMMPMemory */
+
+	return 0;
+}
+
+int
+qat_ae_start(struct qat_softc *sc)
+{
+	int error;
+	u_char ae;
+
+	for (ae = 0; ae < sc->sc_ae_num; ae++) {
+		if ((sc->sc_ae_mask & (1 << ae)) == 0)
+			continue;
+
+		error = qat_aefw_start(sc, ae, 0xff);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
+void
+qat_ae_cluster_intr(void *arg)
+{
+	/* Nothing to implement until we support SRIOV. */
+	printf("qat_ae_cluster_intr\n");
+}
+
+static int
+qat_ae_clear_reset(struct qat_softc *sc)
+{
+	int error;
+	uint32_t times, reset, clock, reg, mask;
+	u_char ae;
+
+	reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
+	reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK));
+	reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK));
+	times = TIMEOUT_AE_RESET;
+	do {
+		qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset);
+		if ((times--) == 0) {
+			device_printf(sc->sc_dev, "couldn't reset AEs\n");
+			return EBUSY;
+		}
+		reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET);
+	} while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) |
+	    __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK))
+	    & reg);
+
+	/* Enable clock for AE and QAT */
+	clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN);
+	clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK);
+	clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK);
+	qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock);
+
+	error = qat_ae_check(sc);
+	if (error)
+		return error;
+
+	/*
+	 * Set undefined power-up/reset states to reasonable default values...
+	 * just to make sure we're starting from a known point
+	 */
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		/* init the ctx_enable */
+		qat_ae_write_4(sc, ae, CTX_ENABLES,
+		    CTX_ENABLES_INIT);
+
+		/* initialize the PCs */
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_STS_INDIRECT,
+		    UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
+
+		/* init the ctx_arb */
+		qat_ae_write_4(sc, ae, CTX_ARB_CNTL,
+		    CTX_ARB_CNTL_INIT);
+
+		/* enable cc */
+		qat_ae_write_4(sc, ae, CC_ENABLE,
+		    CC_ENABLE_INIT);
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_WAKEUP_EVENTS_INDIRECT,
+		    CTX_WAKEUP_EVENTS_INDIRECT_INIT);
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_SIG_EVENTS_INDIRECT,
+		    CTX_SIG_EVENTS_INDIRECT_INIT);
+	}
+
+	if ((sc->sc_ae_mask != 0) &&
+	    sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) {
+		/* XXX XXX XXX init eSram only when this is boot time */
+	}
+
+	if ((sc->sc_ae_mask != 0) &&
+	    sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) {
+		/* XXX XXX XXX wait shram to complete initialization */
+	}
+
+	qat_ae_reset_timestamp(sc);
+
+	return 0;
+}
+
+static int
+qat_ae_check(struct qat_softc *sc)
+{
+	int error, times, ae;
+	uint32_t cnt, pcnt, mask;
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		times = TIMEOUT_AE_CHECK;
+		error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt);
+		if (error) {
+			device_printf(sc->sc_dev,
+			    "couldn't access AE %d CSR\n", ae);
+			return error;
+		}
+		pcnt = cnt & 0xffff;
+
+		while (1) {
+			error = qat_ae_read_4(sc, ae,
+			    PROFILE_COUNT, &cnt);
+			if (error) {
+				device_printf(sc->sc_dev,
+				    "couldn't access AE %d CSR\n", ae);
+				return error;
+			}
+			cnt &= 0xffff;
+			if (cnt == pcnt)
+				times--;
+			else
+				break;
+			if (times <= 0) {
+				device_printf(sc->sc_dev,
+				    "AE %d CSR is useless\n", ae);
+				return EFAULT;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int
+qat_ae_reset_timestamp(struct qat_softc *sc)
+{
+	uint32_t misc, mask;
+	u_char ae;
+
+	/* stop the timestamp timers */
+	misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC);
+	if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) {
+		qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
+		    misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN));
+	}
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0);
+		qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0);
+	}
+
+	/* start timestamp timers */
+	qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC,
+	    misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN);
+
+	return 0;
+}
+
+static void
+qat_ae_clear_xfer(struct qat_softc *sc)
+{
+	u_int mask, reg;
+	u_char ae;
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		for (reg = 0; reg < MAX_GPR_REG; reg++) {
+			qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS,
+			    reg, 0);
+			qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS,
+			    reg, 0);
+		}
+	}
+}
+
+static int
+qat_ae_clear_gprs(struct qat_softc *sc)
+{
+	uint32_t val;
+	uint32_t saved_ctx = 0;
+	int times = TIMEOUT_AE_CHECK, rv;
+	u_char ae;
+	u_int mask;
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		/* turn off share control store bit */
+		val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val);
+		val &= ~AE_MISC_CONTROL_SHARE_CS;
+		qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val);
+
+		/* turn off ucode parity */
+		/* make sure nn_mode is set to self */
+		qat_ae_read_4(sc, ae, CTX_ENABLES, &val);
+		val &= CTX_ENABLES_IGNORE_W1C_MASK;
+		val |= CTX_ENABLES_NN_MODE;
+		val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE;
+		qat_ae_write_4(sc, ae, CTX_ENABLES, val);
+
+		/* copy instructions to ustore */
+		qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst),
+		    ae_clear_gprs_inst);
+
+		/* set PC */
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT,
+		    UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
+
+		/* save current context */
+		qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx);
+		/* change the active context */
+		/* start the context from ctx 0 */
+		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0);
+
+		/* wakeup-event voluntary */
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_WAKEUP_EVENTS_INDIRECT,
+		    CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
+		/* clean signals */
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_SIG_EVENTS_INDIRECT, 0);
+		qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);
+
+		qat_ae_enable_ctx(sc, ae, AE_ALL_CTX);
+	}
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		/* wait for AE to finish */
+		do {
+			rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1);
+		} while (rv && times--);
+		if (times <= 0) {
+			device_printf(sc->sc_dev,
+			    "qat_ae_clear_gprs timeout");
+			return ETIMEDOUT;
+		}
+		qat_ae_disable_ctx(sc, ae, AE_ALL_CTX);
+		/* change the active context */
+		qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
+		    saved_ctx & ACTIVE_CTX_STATUS_ACNO);
+		/* init the ctx_enable */
+		qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT);
+		/* initialize the PCs */
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT);
+		/* init the ctx_arb */
+		qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT);
+		/* enable cc */
+		qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT);
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX,
+		    CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT);
+		qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT,
+		    CTX_SIG_EVENTS_INDIRECT_INIT);
+	}
+
+	return 0;
+}
+
+static void
+qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae)
+{
+	if (ae & 0x1)
+		*nae = ae - 1;
+	else
+		*nae = ae + 1;
+}
+
+static u_int
+qat_ae_ucode_parity64(uint64_t ucode)
+{
+
+	ucode ^= ucode >> 1;
+	ucode ^= ucode >> 2;
+	ucode ^= ucode >> 4;
+	ucode ^= ucode >> 8;
+	ucode ^= ucode >> 16;
+	ucode ^= ucode >> 32;
+
+	return ((u_int)(ucode & 1));
+}
+
+static uint64_t
+qat_ae_ucode_set_ecc(uint64_t ucode)
+{
+	static const uint64_t
+		bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL,
+		bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL,
+		bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL,
+		bit6mask=0xdaf69a46910ULL;
+
+	/* clear the ecc bits */
+	ucode &= ~(0x7fULL << USTORE_ECC_BIT_0);
+
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) <<
+	    USTORE_ECC_BIT_0;
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) <<
+	    USTORE_ECC_BIT_1;
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) <<
+	    USTORE_ECC_BIT_2;
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) <<
+	    USTORE_ECC_BIT_3;
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) <<
+	    USTORE_ECC_BIT_4;
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) <<
+	    USTORE_ECC_BIT_5;
+	ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) <<
+	    USTORE_ECC_BIT_6;
+
+	return (ucode);
+}
+
+static int
+qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
+	const uint64_t *ucode)
+{
+	uint64_t tmp;
+	uint32_t ustore_addr, ulo, uhi;
+	int i;
+
+	qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
+	uaddr |= USTORE_ADDRESS_ECS;
+
+	qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
+	for (i = 0; i < ninst; i++) {
+		tmp = qat_ae_ucode_set_ecc(ucode[i]);
+		ulo = (uint32_t)(tmp & 0xffffffff);
+		uhi = (uint32_t)(tmp >> 32);
+
+		qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo);
+		/* this will auto increment the address */
+		qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi);
+	}
+	qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);
+
+	return 0;
+}
+
+static int
+qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst,
+    uint64_t *ucode)
+{
+	uint32_t misc, ustore_addr, ulo, uhi;
+	u_int ii;
+	u_char nae;
+
+	if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED)
+		return EBUSY;
+
+	/* determine whether it neighbour AE runs in shared control store
+	 * status */
+	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
+	if (misc & AE_MISC_CONTROL_SHARE_CS) {
+		qat_ae_get_shared_ustore_ae(ae, &nae);
+		if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
+			return EBUSY;
+	}
+
+	/* if reloadable, then get it all from dram-ustore */
+	if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD))
+		panic("notyet"); /* XXX getReloadUwords */
+
+	/* disable SHARE_CS bit to workaround silicon bug */
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb);
+
+	MPASS(uaddr + ninst <= USTORE_SIZE);
+
+	/* save ustore-addr csr */
+	qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr);
+
+	uaddr |= USTORE_ADDRESS_ECS;	/* enable ecs bit */
+	for (ii = 0; ii < ninst; ii++) {
+		qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr);
+
+		uaddr++;
+		qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo);
+		qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi);
+		ucode[ii] = uhi;
+		ucode[ii] = (ucode[ii] << 32) | ulo;
+	}
+
+	/* restore SHARE_CS bit to workaround silicon bug */
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
+	qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr);
+
+	return 0;
+}
+
+static u_int
+qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr,
+    u_int *value)
+{
+	const uint64_t *inst_arr;
+	u_int ninst0, curvalue;
+	int ii, vali, fixup, usize = 0;
+
+	if (size == 0)
+		return 0;
+
+	ninst0 = ninst;
+	vali = 0;
+	curvalue = value[vali++];
+
+	switch (size) {
+	case 0x1:
+		inst_arr = ae_inst_1b;
+		usize = nitems(ae_inst_1b);
+		break;
+	case 0x2:
+		inst_arr = ae_inst_2b;
+		usize = nitems(ae_inst_2b);
+		break;
+	case 0x3:
+		inst_arr = ae_inst_3b;
+		usize = nitems(ae_inst_3b);
+		break;
+	default:
+		inst_arr = ae_inst_4b;
+		usize = nitems(ae_inst_4b);
+		break;
+	}
+
+	fixup = ninst;
+	for (ii = 0; ii < usize; ii++)
+		ucode[ninst++] = inst_arr[ii];
+
+	INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr));
+	fixup++;
+	INSERT_IMMED_GPRA_CONST(ucode[fixup], 0);
+	fixup++;
+	INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
+	fixup++;
+	INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
+	/* XXX fixup++ ? */
+
+	if (size <= 0x4)
+		return (ninst - ninst0);
+
+	size -= sizeof(u_int);
+	while (size >= sizeof(u_int)) {
+		curvalue = value[vali++];
+		fixup = ninst;
+		ucode[ninst++] = ae_inst_4b[0x2];
+		ucode[ninst++] = ae_inst_4b[0x3];
+		ucode[ninst++] = ae_inst_4b[0x8];
+		INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16));
+		fixup++;
+		INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0));
+		/* XXX fixup++ ? */
+
+		addr += sizeof(u_int);
+		size -= sizeof(u_int);
+	}
+	/* call this function recusive when the left size less than 4 */
+	ninst +=
+	    qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali);
+
+	return (ninst - ninst0);
+}
+
+static int
+qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx,
+    uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles,
+    u_int *endpc)
+{
+	int error = 0, share_cs = 0;
+	uint64_t savucode[MAX_EXEC_INST];
+	uint32_t indr_lm_addr_0, indr_lm_addr_1;
+	uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1;
+	uint32_t indr_future_cnt_sig;
+	uint32_t indr_sig, active_sig;
+	uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl;
+	uint32_t misc, nmisc, ctxen;
+	u_char nae;
+
+	MPASS(ninst <= USTORE_SIZE);
+
+	if (qat_ae_is_active(sc, ae))
+		return EBUSY;
+
+	/* save current LM addr */
+	qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0);
+	qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1);
+	qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
+	    &indr_lm_addr_byte_0);
+	qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
+	    &indr_lm_addr_byte_1);
+
+	/* backup shared control store bit, and force AE to
+	   none-shared mode before executing ucode snippet */
+	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
+	if (misc & AE_MISC_CONTROL_SHARE_CS) {
+		share_cs = 1;
+		qat_ae_get_shared_ustore_ae(ae, &nae);
+		if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae))
+			return EBUSY;
+	}
+	nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
+
+	/* save current states: */
+	if (ninst <= MAX_EXEC_INST) {
+		error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode);
+		if (error) {
+			qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc);
+			return error;
+		}
+	}
+
+	/* save wakeup-events */
+	qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT,
+	    &wakeup_ev);
+	/* save PC */
+	qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc);
+	savpc &= UPC_MASK;
+
+	/* save ctx enables */
+	qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen);
+	ctxen &= CTX_ENABLES_IGNORE_W1C_MASK;
+	/* save conditional-code */
+	qat_ae_read_4(sc, ae, CC_ENABLE, &savcc);
+	/* save current context */
+	qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx);
+	qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl);
+
+	/* save indirect csrs */
+	qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
+	    &indr_future_cnt_sig);
+	qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig);
+	qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig);
+
+	/* turn off ucode parity */
+	qat_ae_write_4(sc, ae, CTX_ENABLES,
+	    ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE);
+
+	/* copy instructions to ustore */
+	qat_ae_ucode_write(sc, ae, 0, ninst, ucode);
+	/* set PC */
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0);
+	/* change the active context */
+	qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
+	    ctx & ACTIVE_CTX_STATUS_ACNO);
+
+	if (cond_code_off) {
+		/* disable conditional-code*/
+		qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff);
+	}
+
+	/* wakeup-event voluntary */
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx,
+	    CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY);
+
+	/* clean signals */
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0);
+	qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0);
+
+	/* enable context */
+	qat_ae_enable_ctx(sc, ae, 1 << ctx);
+
+	/* wait for it to finish */
+	if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0)
+		error = ETIMEDOUT;
+
+	/* see if we need to get the current PC */
+	if (endpc != NULL) {
+		uint32_t ctx_status;
+
+		qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
+		    &ctx_status);
+		*endpc = ctx_status & UPC_MASK;
+	}
+#if 0
+	{
+		uint32_t ctx_status;
+
+		qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT,
+		    &ctx_status);
+		printf("%s: endpc 0x%08x\n", __func__,
+		    ctx_status & UPC_MASK);
+	}
+#endif
+
+	/* retore to previous states: */
+	/* disable context */
+	qat_ae_disable_ctx(sc, ae, 1 << ctx);
+	if (ninst <= MAX_EXEC_INST) {
+		/* instructions */
+		qat_ae_ucode_write(sc, ae, 0, ninst, savucode);
+	}
+	/* wakeup-events */
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT,
+	    wakeup_ev);
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc);
+
+	/* only restore shared control store bit,
+	   other bit might be changed by AE code snippet */
+	qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc);
+	if (share_cs)
+		nmisc = misc | AE_MISC_CONTROL_SHARE_CS;
+	else
+		nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS;
+	qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc);
+	/* conditional-code */
+	qat_ae_write_4(sc, ae, CC_ENABLE, savcc);
+	/* change the active context */
+	qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS,
+	    savctx & ACTIVE_CTX_STATUS_ACNO);
+	/* restore the nxt ctx to run */
+	qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl);
+	/* restore current LM addr */
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT,
+	    indr_lm_addr_0);
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT,
+	    indr_lm_addr_1);
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
+	    indr_lm_addr_byte_0);
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
+	    indr_lm_addr_byte_1);
+
+	/* restore indirect csrs */
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
+	    indr_future_cnt_sig);
+	qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT,
+	    indr_sig);
+	qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig);
+
+	/* ctx-enables */
+	qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen);
+
+	return error;
+}
+
+static int
+qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx,
+    int *first_exec, uint64_t *ucode, u_int ninst,
+    u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1)
+{
+
+	if (*first_exec) {
+		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
+		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
+		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
+		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
+		qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
+		*first_exec = 0;
+	}
+
+	return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL);
+}
+
+static int
+qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx,
+    u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1)
+{
+	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0);
+	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1);
+	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2);
+	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0);
+	qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1);
+
+	return 0;
+}
+
+static int
+qat_ae_get_inst_num(int lmsize)
+{
+	int ninst, left;
+
+	if (lmsize == 0)
+		return 0;
+
+	left = lmsize % sizeof(u_int);
+
+	if (left) {
+		ninst = nitems(ae_inst_1b) +
+		    qat_ae_get_inst_num(lmsize - left);
+	} else {
+		/* 3 instruction is needed for further code */
+		ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b);
+	}
+
+	return (ninst);
+}
+
+static int
+qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae,
+    struct qat_ae_batch_init_list *qabi_list, size_t nqabi)
+{
+	struct qat_ae_batch_init *qabi;
+	size_t alloc_ninst, ninst;
+	uint64_t *ucode;
+	u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1;
+	int insnsz, error = 0, execed = 0, first_exec = 1;
+
+	if (STAILQ_FIRST(qabi_list) == NULL)
+		return 0;
+
+	alloc_ninst = min(USTORE_SIZE, nqabi);
+	ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst);
+
+	ninst = 0;
+	STAILQ_FOREACH(qabi, qabi_list, qabi_next) {
+		insnsz = qat_ae_get_inst_num(qabi->qabi_size);
+		if (insnsz + ninst > alloc_ninst) {
+			/* add ctx_arb[kill] */
+			ucode[ninst++] = 0x0E000010000ull;
+			execed = 1;
+
+			error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
+			    &first_exec, ucode, ninst,
+			    &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
+			if (error) {
+				qat_ae_restore_init_lm_gprs(sc, ae, 0,
+				    gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
+				qat_free_mem(ucode);
+				return error;
+			}
+			/* run microExec to execute the microcode */
+			ninst = 0;
+		}
+		ninst += qat_ae_concat_ucode(ucode, ninst,
+		    qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value);
+	}
+
+	if (ninst > 0) {
+		ucode[ninst++] = 0x0E000010000ull;
+		execed = 1;
+
+		error = qat_ae_exec_ucode_init_lm(sc, ae, 0,
+		    &first_exec, ucode, ninst,
+		    &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1);
+	}
+	if (execed) {
+		qat_ae_restore_init_lm_gprs(sc, ae, 0,
+		    gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1);
+	}
+
+	qat_free_mem(ucode);
+
+	return error;
+}
+
+static int
+qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc)
+{
+
+	if (qat_ae_is_active(sc, ae))
+		return EBUSY;
+
+	qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT,
+	    UPC_MASK & upc);
+	return 0;
+}
+
+static inline u_int
+qat_aefw_csum_calc(u_int reg, int ch)
+{
+	int i;
+	u_int topbit = CRC_BITMASK(CRC_WIDTH - 1);
+	u_int inbyte = (u_int)((reg >> 0x18) ^ ch);
+
+	reg ^= inbyte << (CRC_WIDTH - 0x8);
+	for (i = 0; i < 0x8; i++) {
+		if (reg & topbit)
+			reg = (reg << 1) ^ CRC_POLY;
+		else
+			reg <<= 1;
+	}
+
+	return (reg & CRC_WIDTHMASK(CRC_WIDTH));
+}
+
+static u_int
+qat_aefw_csum(char *buf, int size)
+{
+	u_int csum = 0;
+
+	while (size--) {
+		csum = qat_aefw_csum_calc(csum, *buf++);
+	}
+
+	return csum;
+}
+
+static const char *
+qat_aefw_uof_string(struct qat_softc *sc, size_t offset)
+{
+	if (offset >= sc->sc_aefw_uof.qafu_str_tab_size)
+		return NULL;
+	if (sc->sc_aefw_uof.qafu_str_tab == NULL)
+		return NULL;
+
+	return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset);
+}
+
+static struct uof_chunk_hdr *
+qat_aefw_uof_find_chunk(struct qat_softc *sc,
+	const char *id, struct uof_chunk_hdr *cur)
+{
+	struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr;
+	struct uof_chunk_hdr *uch;
+	int i;
+
+	uch = (struct uof_chunk_hdr *)(uoh + 1);
+	for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) {
+		if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size)
+			return NULL;
+
+		if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN))
+			return uch;
+	}
+
+	return NULL;
+}
+
+static int
+qat_aefw_load_mof(struct qat_softc *sc)
+{
+	const struct firmware *fw;
+
+	fw = firmware_get(sc->sc_hw.qhw_mof_fwname);
+	if (fw == NULL) {
+		device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
+		    sc->sc_hw.qhw_mof_fwname);
+		return ENXIO;
+	}
+
+	sc->sc_fw_mof = qat_alloc_mem(fw->datasize);
+	sc->sc_fw_mof_size = fw->datasize;
+	memcpy(sc->sc_fw_mof, fw->data, fw->datasize);
+	firmware_put(fw, FIRMWARE_UNLOAD);
+	return 0;
+}
+
+static void
+qat_aefw_unload_mof(struct qat_softc *sc)
+{
+	if (sc->sc_fw_mof != NULL) {
+		qat_free_mem(sc->sc_fw_mof);
+		sc->sc_fw_mof = NULL;
+	}
+}
+
+static int
+qat_aefw_load_mmp(struct qat_softc *sc)
+{
+	const struct firmware *fw;
+
+	fw = firmware_get(sc->sc_hw.qhw_mmp_fwname);
+	if (fw == NULL) {
+		device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n",
+		    sc->sc_hw.qhw_mmp_fwname);
+		return ENXIO;
+	}
+
+	sc->sc_fw_mmp = qat_alloc_mem(fw->datasize);
+	sc->sc_fw_mmp_size = fw->datasize;
+	memcpy(sc->sc_fw_mmp, fw->data, fw->datasize);
+	firmware_put(fw, FIRMWARE_UNLOAD);
+	return 0;
+}
+
+static void
+qat_aefw_unload_mmp(struct qat_softc *sc)
+{
+	if (sc->sc_fw_mmp != NULL) {
+		qat_free_mem(sc->sc_fw_mmp);
+		sc->sc_fw_mmp = NULL;
+	}
+}
+
+static int
+qat_aefw_mof_find_uof0(struct qat_softc *sc,
+	struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head,
+	u_int nchunk, size_t size, const char *id,
+	size_t *fwsize, void **fwptr)
+{
+	int i;
+	char *uof_name;
+
+	for (i = 0; i < nchunk; i++) {
+		struct mof_uof_chunk_hdr *much = &head[i];
+
+		if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN))
+			return EINVAL;
+
+		if (much->much_offset + much->much_size > size)
+			return EINVAL;
+
+		if (sc->sc_mof.qmf_sym_size <= much->much_name)
+			return EINVAL;
+
+		uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym +
+		    much->much_name);
+
+		if (!strcmp(uof_name, sc->sc_fw_uof_name)) {
+			*fwptr = (void *)((uintptr_t)muh +
+			    (uintptr_t)much->much_offset);
+			*fwsize = (size_t)much->much_size;
+			return 0;
+		}
+	}
+
+	return ENOENT;
+}
+
+static int
+qat_aefw_mof_find_uof(struct qat_softc *sc)
+{
+	struct mof_uof_hdr *uof_hdr, *suof_hdr;
+	u_int nuof_chunks = 0, nsuof_chunks = 0;
+	int error;
+
+	uof_hdr = sc->sc_mof.qmf_uof_objs;
+	suof_hdr = sc->sc_mof.qmf_suof_objs;
+
+	if (uof_hdr != NULL) {
+		if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) {
+			return EINVAL;
+		}
+		nuof_chunks = uof_hdr->muh_num_chunks;
+	}
+	if (suof_hdr != NULL) {
+		if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks)
+			return EINVAL;
+		nsuof_chunks = suof_hdr->muh_num_chunks;
+	}
+
+	if (nuof_chunks + nsuof_chunks == 0)
+		return EINVAL;
+
+	if (uof_hdr != NULL) {
+		error = qat_aefw_mof_find_uof0(sc, uof_hdr,
+		    (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks,
+		    sc->sc_mof.qmf_uof_objs_size, UOF_IMAG,
+		    &sc->sc_fw_uof_size, &sc->sc_fw_uof);
+		if (error && error != ENOENT)
+			return error;
+	}
+
+	if (suof_hdr != NULL) {
+		error = qat_aefw_mof_find_uof0(sc, suof_hdr,
+		    (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks,
+		    sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG,
+		    &sc->sc_fw_suof_size, &sc->sc_fw_suof);
+		if (error && error != ENOENT)
+			return error;
+	}
+
+	if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL)
+		return ENOENT;
+
+	return 0;
+}
+
+static int
+qat_aefw_mof_parse(struct qat_softc *sc)
+{
+	const struct mof_file_hdr *mfh;
+	const struct mof_file_chunk_hdr *mfch;
+	size_t size;
+	u_int csum;
+	int error, i;
+
+	size = sc->sc_fw_mof_size;
+
+	if (size < sizeof(struct mof_file_hdr))
+		return EINVAL;
+	size -= sizeof(struct mof_file_hdr);
+
+	mfh = sc->sc_fw_mof;
+
+	if (mfh->mfh_fid != MOF_FID)
+		return EINVAL;
+
+	csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof +
+	    offsetof(struct mof_file_hdr, mfh_min_ver)),
+	    sc->sc_fw_mof_size -
+	    offsetof(struct mof_file_hdr, mfh_min_ver));
+	if (mfh->mfh_csum != csum)
+		return EINVAL;
+
+	if (mfh->mfh_min_ver != MOF_MIN_VER ||
+	    mfh->mfh_maj_ver != MOF_MAJ_VER)
+		return EINVAL;
+
+	if (mfh->mfh_max_chunks < mfh->mfh_num_chunks)
+		return EINVAL;
+
+	if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks)
+		return EINVAL;
+	mfch = (const struct mof_file_chunk_hdr *)(mfh + 1);
+
+	for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) {
+		if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size)
+			return EINVAL;
+
+		if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) {
+			if (sc->sc_mof.qmf_sym != NULL)
+				return EINVAL;
+
+			sc->sc_mof.qmf_sym =
+			    (void *)((uintptr_t)sc->sc_fw_mof +
+			    (uintptr_t)mfch->mfch_offset + sizeof(u_int));
+			sc->sc_mof.qmf_sym_size =
+			    *(u_int *)((uintptr_t)sc->sc_fw_mof +
+			    (uintptr_t)mfch->mfch_offset);
+
+			if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0)
+				return EINVAL;
+			if (mfch->mfch_size != sc->sc_mof.qmf_sym_size +
+			    sizeof(u_int) || mfch->mfch_size == 0)
+				return EINVAL;
+			if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym +
+			    sc->sc_mof.qmf_sym_size - 1) != '\0')
+				return EINVAL;
+
+		} else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) {
+			if (sc->sc_mof.qmf_uof_objs != NULL)
+				return EINVAL;
+
+			sc->sc_mof.qmf_uof_objs =
+			    (void *)((uintptr_t)sc->sc_fw_mof +
+			    (uintptr_t)mfch->mfch_offset);
+			sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size;
+
+		} else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) {
+			if (sc->sc_mof.qmf_suof_objs != NULL)
+				return EINVAL;
+
+			sc->sc_mof.qmf_suof_objs =
+			    (void *)((uintptr_t)sc->sc_fw_mof +
+			    (uintptr_t)mfch->mfch_offset);
+			sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size;
+		}
+	}
+
+	if (sc->sc_mof.qmf_sym == NULL ||
+	    (sc->sc_mof.qmf_uof_objs == NULL &&
+	    sc->sc_mof.qmf_suof_objs == NULL))
+		return EINVAL;
+
+	error = qat_aefw_mof_find_uof(sc);
+	if (error)
+		return error;
+	return 0;
+}
+
+static int
+qat_aefw_uof_parse_image(struct qat_softc *sc,
+	struct qat_uof_image *qui, struct uof_chunk_hdr *uch)
+{
+	struct uof_image *image;
+	struct uof_code_page *page;
+	uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
+	size_t lim = uch->uch_offset + uch->uch_size, size;
+	int i, p;
+
+	size = uch->uch_size;
+	if (size < sizeof(struct uof_image))
+		return EINVAL;
+	size -= sizeof(struct uof_image);
+
+	qui->qui_image = image =
+	    (struct uof_image *)(base + uch->uch_offset);
+
+#define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim)			\
+do {									\
+	u_int nent;							\
+	nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\
+	if ((lim) < off + sizeof(struct uof_obj_table) +		\
+	    sizeof(type) * nent)					\
+		return EINVAL;						\
+	*(np) = nent;							\
+	if (nent > 0)							\
+		*(typep) = (type)((struct uof_obj_table *)		\
+		    ((base) + (off)) + 1);				\
+	else								\
+		*(typep) = NULL;					\
+} while (0)
+
+	ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg,
+	    struct uof_ae_reg *, base, image->ui_reg_tab, lim);
+	ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym,
+	    struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim);
+	ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak,
+	    struct qui_sbreak *, base, image->ui_sbreak_tab, lim);
+
+	if (size < sizeof(struct uof_code_page) * image->ui_num_pages)
+		return EINVAL;
+	if (nitems(qui->qui_pages) < image->ui_num_pages)
+		return EINVAL;
+
+	page = (struct uof_code_page *)(image + 1);
+
+	for (p = 0; p < image->ui_num_pages; p++, page++) {
+		struct qat_uof_page *qup = &qui->qui_pages[p];
+		struct uof_code_area *uca;
+
+		qup->qup_page_num = page->ucp_page_num;
+		qup->qup_def_page = page->ucp_def_page;
+		qup->qup_page_region = page->ucp_page_region;
+		qup->qup_beg_vaddr = page->ucp_beg_vaddr;
+		qup->qup_beg_paddr = page->ucp_beg_paddr;
+
+		ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var,
+		    struct uof_uword_fixup *, base,
+		    page->ucp_uc_var_tab, lim);
+		ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var,
+		    struct uof_import_var *, base,
+		    page->ucp_imp_var_tab, lim);
+		ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr,
+		    struct uof_uword_fixup *, base,
+		    page->ucp_imp_expr_tab, lim);
+		ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg,
+		    struct uof_uword_fixup *, base,
+		    page->ucp_neigh_reg_tab, lim);
+
+		if (lim < page->ucp_code_area + sizeof(struct uof_code_area))
+			return EINVAL;
+
+		uca = (struct uof_code_area *)(base + page->ucp_code_area);
+		qup->qup_num_micro_words = uca->uca_num_micro_words;
+
+		ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks,
+		    struct qat_uof_uword_block *, base,
+		    uca->uca_uword_block_tab, lim);
+
+		for (i = 0; i < qup->qup_num_uw_blocks; i++) {
+			u_int uwordoff = ((struct uof_uword_block *)(
+			    &qup->qup_uw_blocks[i]))->uub_uword_offset;
+
+			if (lim < uwordoff)
+				return EINVAL;
+
+			qup->qup_uw_blocks[i].quub_micro_words =
+			    (base + uwordoff);
+		}
+	}
+
+#undef ASSIGN_OBJ_TAB
+
+	return 0;
+}
+
+static int
+qat_aefw_uof_parse_images(struct qat_softc *sc)
+{
+	struct uof_chunk_hdr *uch = NULL;
+	u_int assigned_ae;
+	int i, error;
+
+	for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) {
+		uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch);
+		if (uch == NULL)
+			break;
+
+		if (i >= nitems(sc->sc_aefw_uof.qafu_imgs))
+			return ENOENT;
+
+		error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch);
+		if (error)
+			return error;
+
+		sc->sc_aefw_uof.qafu_num_imgs++;
+	}
+
+	assigned_ae = 0;
+	for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
+		assigned_ae |= sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned;
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_uof_parse(struct qat_softc *sc)
+{
+	struct uof_file_hdr *ufh;
+	struct uof_file_chunk_hdr *ufch;
+	struct uof_obj_hdr *uoh;
+	struct uof_chunk_hdr *uch;
+	void *uof = NULL;
+	size_t size, uof_size, hdr_size;
+	uintptr_t base;
+	u_int csum;
+	int i;
+
+	size = sc->sc_fw_uof_size;
+	if (size < MIN_UOF_SIZE)
+		return EINVAL;
+	size -= sizeof(struct uof_file_hdr);
+
+	ufh = sc->sc_fw_uof;
+
+	if (ufh->ufh_id != UOF_FID)
+		return EINVAL;
+	if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER)
+		return EINVAL;
+
+	if (ufh->ufh_max_chunks < ufh->ufh_num_chunks)
+		return EINVAL;
+	if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks)
+		return EINVAL;
+	ufch = (struct uof_file_chunk_hdr *)(ufh + 1);
+
+	uof_size = 0;
+	for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) {
+		if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size)
+			return EINVAL;
+
+		if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) {
+			if (uof != NULL)
+				return EINVAL;
+
+			uof =
+			    (void *)((uintptr_t)sc->sc_fw_uof +
+			    ufch->ufch_offset);
+			uof_size = ufch->ufch_size;
+
+			csum = qat_aefw_csum(uof, uof_size);
+			if (csum != ufch->ufch_csum)
+				return EINVAL;
+		}
+	}
+
+	if (uof == NULL)
+		return ENOENT;
+
+	size = uof_size;
+	if (size < sizeof(struct uof_obj_hdr))
+		return EINVAL;
+	size -= sizeof(struct uof_obj_hdr);
+
+	uoh = uof;
+
+	if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks)
+		return EINVAL;
+
+	/* Check if the UOF objects are compatible with the chip */
+	if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0)
+		return ENOTSUP;
+
+	if (uoh->uoh_min_cpu_ver > sc->sc_rev ||
+	    uoh->uoh_max_cpu_ver < sc->sc_rev)
+		return ENOTSUP;
+
+	sc->sc_aefw_uof.qafu_size = uof_size;
+	sc->sc_aefw_uof.qafu_obj_hdr = uoh;
+
+	base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr;
+
+	/* map uof string-table */
+	uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL);
+	if (uch != NULL) {
+		hdr_size = offsetof(struct uof_str_tab, ust_strings);
+		sc->sc_aefw_uof.qafu_str_tab =
+		    (void *)(base + uch->uch_offset + hdr_size);
+		sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size;
+	}
+
+	/* get ustore mem inits table -- should be only one */
+	uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL);
+	if (uch != NULL) {
+		if (uch->uch_size < sizeof(struct uof_obj_table))
+			return EINVAL;
+		sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base +
+		    uch->uch_offset))->uot_nentries;
+		if (sc->sc_aefw_uof.qafu_num_init_mem) {
+			sc->sc_aefw_uof.qafu_init_mem =
+			    (struct uof_init_mem *)(base + uch->uch_offset +
+			    sizeof(struct uof_obj_table));
+			sc->sc_aefw_uof.qafu_init_mem_size =
+			    uch->uch_size - sizeof(struct uof_obj_table);
+		}
+	}
+
+	uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL);
+	if (uch != NULL) {
+		if (uch->uch_size < sizeof(struct uof_obj_table) +
+		    sizeof(struct uof_var_mem_seg))
+			return EINVAL;
+		sc->sc_aefw_uof.qafu_var_mem_seg =
+		    (struct uof_var_mem_seg *)(base + uch->uch_offset +
+		    sizeof(struct uof_obj_table));
+	}
+
+	return qat_aefw_uof_parse_images(sc);
+}
+
+static int
+qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi,
+    struct suof_chunk_hdr *sch)
+{
+	struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
+	struct simg_ae_mode *ae_mode;
+	u_int maj_ver;
+
+	qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset +
+	    sizeof(struct suof_obj_hdr);
+	qsi->qsi_simg_len =
+	    ((struct suof_obj_hdr *)
+	    (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length;
+
+	qsi->qsi_css_header = qsi->qsi_simg_buf;
+	qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr);
+	qsi->qsi_css_signature = qsi->qsi_css_key +
+	    CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
+	qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN;
+
+	ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg;
+	qsi->qsi_ae_mask = ae_mode->sam_ae_mask;
+	qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name;
+	qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data;
+	qsi->qsi_fw_type = ae_mode->sam_fw_type;
+
+	if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type)
+		return EINVAL;
+
+	maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff;
+	if ((maj_ver > ae_mode->sam_devmax_ver) ||
+	    (maj_ver < ae_mode->sam_devmin_ver)) {
+		return EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_suof_parse(struct qat_softc *sc)
+{
+	struct suof_file_hdr *sfh;
+	struct suof_chunk_hdr *sch;
+	struct qat_aefw_suof *qafs = &sc->sc_aefw_suof;
+	struct qat_suof_image *qsi;
+	size_t size;
+	u_int csum;
+	int ae0_img = MAX_AE;
+	int i, error;
+
+	size = sc->sc_fw_suof_size;
+	if (size < sizeof(struct suof_file_hdr))
+		return EINVAL;
+
+	sfh = sc->sc_fw_suof;
+
+	if (sfh->sfh_file_id != SUOF_FID)
+		return EINVAL;
+	if (sfh->sfh_fw_type != 0)
+		return EINVAL;
+	if (sfh->sfh_num_chunks <= 1)
+		return EINVAL;
+	if (sfh->sfh_min_ver != SUOF_MIN_VER ||
+	    sfh->sfh_maj_ver != SUOF_MAJ_VER)
+		return EINVAL;
+
+	csum = qat_aefw_csum((char *)&sfh->sfh_min_ver,
+	    size - offsetof(struct suof_file_hdr, sfh_min_ver));
+	if (csum != sfh->sfh_check_sum)
+		return EINVAL;
+
+	size -= sizeof(struct suof_file_hdr);
+
+	qafs->qafs_file_id = SUOF_FID;
+	qafs->qafs_suof_buf = sc->sc_fw_suof;
+	qafs->qafs_suof_size = sc->sc_fw_suof_size;
+	qafs->qafs_check_sum = sfh->sfh_check_sum;
+	qafs->qafs_min_ver = sfh->sfh_min_ver;
+	qafs->qafs_maj_ver = sfh->sfh_maj_ver;
+	qafs->qafs_fw_type = sfh->sfh_fw_type;
+
+	if (size < sizeof(struct suof_chunk_hdr))
+		return EINVAL;
+	sch = (struct suof_chunk_hdr *)(sfh + 1);
+	size -= sizeof(struct suof_chunk_hdr);
+
+	if (size < sizeof(struct suof_str_tab))
+		return EINVAL;
+	size -= offsetof(struct suof_str_tab, sst_strings);
+
+	qafs->qafs_sym_size = ((struct suof_str_tab *)
+	    (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length;
+	if (size < qafs->qafs_sym_size)
+		return EINVAL;
+	qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset +
+	    offsetof(struct suof_str_tab, sst_strings);
+
+	qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1;
+	if (qafs->qafs_num_simgs == 0)
+		return EINVAL;
+
+	qsi = qat_alloc_mem(
+	    sizeof(struct qat_suof_image) * qafs->qafs_num_simgs);
+	qafs->qafs_simg = qsi;
+
+	for (i = 0; i < qafs->qafs_num_simgs; i++) {
+		error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]);
+		if (error)
+			return error;
+		if ((qsi[i].qsi_ae_mask & 0x1) != 0)
+			ae0_img = i;
+	}
+
+	if (ae0_img != qafs->qafs_num_simgs - 1) {
+		struct qat_suof_image last_qsi;
+
+		memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1],
+		    sizeof(struct qat_suof_image));
+		memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img],
+		    sizeof(struct qat_suof_image));
+		memcpy(&qsi[ae0_img], &last_qsi,
+		    sizeof(struct qat_suof_image));
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size,
+    struct qat_dmamem *dma)
+{
+	struct css_hdr *css = (struct css_hdr *)image;
+	struct auth_chunk *auth_chunk;
+	struct fw_auth_desc *auth_desc;
+	size_t mapsize, simg_offset = sizeof(struct auth_chunk);
+	bus_size_t bus_addr;
+	uintptr_t virt_addr;
+	int error;
+
+	if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN)
+		return EINVAL;
+
+	mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ?
+	    CSS_AE_SIMG_LEN + simg_offset :
+	    size + CSS_FWSK_PAD_LEN + simg_offset;
+	error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE);
+	if (error)
+		return error;
+
+	memset(dma->qdm_dma_vaddr, 0, mapsize);
+
+	auth_chunk = dma->qdm_dma_vaddr;
+	auth_chunk->ac_chunk_size = mapsize;
+	auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr;
+
+	virt_addr = (uintptr_t)dma->qdm_dma_vaddr;
+	virt_addr += simg_offset;
+	bus_addr = auth_chunk->ac_chunk_bus_addr;
+	bus_addr += simg_offset;
+
+	auth_desc = &auth_chunk->ac_fw_auth_desc;
+	auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32;
+	auth_desc->fad_css_hdr_low = bus_addr;
+
+	memcpy((void *)virt_addr, image, sizeof(struct css_hdr));
+	/* pub key */
+	virt_addr += sizeof(struct css_hdr);
+	bus_addr += sizeof(struct css_hdr);
+	image += sizeof(struct css_hdr);
+
+	auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32;
+	auth_desc->fad_fwsk_pub_low = bus_addr;
+
+	memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN);
+	memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN);
+	memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN),
+	    image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t));
+
+	virt_addr += CSS_FWSK_PUB_LEN;
+	bus_addr += CSS_FWSK_PUB_LEN;
+	image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN;
+
+	auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32;
+	auth_desc->fad_signature_low = bus_addr;
+
+	memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN);
+
+	virt_addr += CSS_SIGNATURE_LEN;
+	bus_addr += CSS_SIGNATURE_LEN;
+	image += CSS_SIGNATURE_LEN;
+
+	auth_desc->fad_img_high = (uint64_t)bus_addr >> 32;
+	auth_desc->fad_img_low = bus_addr;
+	auth_desc->fad_img_len = size - AE_IMG_OFFSET;
+
+	memcpy((void *)virt_addr, image, auth_desc->fad_img_len);
+
+	if (css->css_fw_type == CSS_AE_FIRMWARE) {
+		auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high;
+		auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low;
+
+		bus_addr += sizeof(struct simg_ae_mode);
+
+		auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32;
+		auth_desc->fad_img_ae_init_data_low = bus_addr;
+
+		bus_addr += SIMG_AE_INIT_SEQ_LEN;
+
+		auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32;
+		auth_desc->fad_img_ae_insts_low = bus_addr;
+	} else {
+		auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high;
+		auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low;
+	}
+
+	bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map,
+	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
+
+	return 0;
+}
+
+static int
+qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma)
+{
+	bus_addr_t addr;
+	uint32_t fcu, sts;
+	int retry = 0;
+
+	addr = dma->qdm_dma_seg.ds_addr;
+	qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32);
+	qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr);
+	qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH);
+
+	do {
+		DELAY(FW_AUTH_WAIT_PERIOD * 1000);
+		fcu = qat_cap_global_read_4(sc, FCU_STATUS);
+		sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
+		if (sts == FCU_STATUS_STS_VERI_FAIL)
+			goto fail;
+		if (fcu & FCU_STATUS_AUTHFWLD &&
+		    sts == FCU_STATUS_STS_VERI_DONE) {
+			return 0;
+		}
+	} while (retry++ < FW_AUTH_MAX_RETRY);
+
+fail:
+	device_printf(sc->sc_dev,
+	   "firmware authentication error: status 0x%08x retry %d\n",
+	   fcu, retry);
+	return EINVAL;
+}
+
+static int
+qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma)
+{
+	struct simg_ae_mode *ae_mode;
+	uint32_t fcu, sts, loaded;
+	u_int mask;
+	u_char ae;
+	int retry = 0;
+
+	ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr +
+	    sizeof(struct auth_chunk) + sizeof(struct css_hdr) +
+	    CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN);
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		if (!((ae_mode->sam_ae_mask >> ae) & 0x1))
+			continue;
+		if (qat_ae_is_active(sc, ae)) {
+			device_printf(sc->sc_dev, "AE %d is active\n", ae);
+			return EINVAL;
+		}
+		qat_cap_global_write_4(sc, FCU_CTRL,
+		    FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE));
+		do {
+			DELAY(FW_AUTH_WAIT_PERIOD * 1000);
+			fcu = qat_cap_global_read_4(sc, FCU_STATUS);
+			sts = __SHIFTOUT(fcu, FCU_STATUS_STS);
+			loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE);
+			if (sts == FCU_STATUS_STS_LOAD_DONE &&
+			    (loaded & (1 << ae))) {
+				break;
+			}
+		} while (retry++ < FW_AUTH_MAX_RETRY);
+
+		if (retry > FW_AUTH_MAX_RETRY) {
+			device_printf(sc->sc_dev,
+			    "firmware load timeout: status %08x\n", fcu);
+			return EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_suof_write(struct qat_softc *sc)
+{
+	struct qat_suof_image *qsi = NULL;
+	int i, error = 0;
+
+	for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) {
+		qsi = &sc->sc_aefw_suof.qafs_simg[i];
+		error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf,
+		    qsi->qsi_simg_len, &qsi->qsi_dma);
+		if (error)
+			return error;
+		error = qat_aefw_auth(sc, &qsi->qsi_dma);
+		if (error)
+			goto fail;
+		error = qat_aefw_suof_load(sc, &qsi->qsi_dma);
+		if (error)
+			goto fail;
+
+		qat_free_dmamem(sc, &qsi->qsi_dma);
+	}
+	qat_free_mem(sc->sc_aefw_suof.qafs_simg);
+
+	return 0;
+fail:
+	if (qsi != NULL)
+		qat_free_dmamem(sc, &qsi->qsi_dma);
+	return error;
+}
+
+static int
+qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae,
+	struct qat_uof_image *qui)
+{
+	struct qat_ae_slice *slice;
+	int i, npages, nregions;
+
+	if (qae->qae_num_slices >= nitems(qae->qae_slices))
+		return ENOENT;
+
+	if (qui->qui_image->ui_ae_mode &
+	    (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) {
+		/* XXX */
+		device_printf(sc->sc_dev,
+		    "shared ae mode is not supported yet\n");
+		return ENOTSUP;
+	}
+
+	qae->qae_shareable_ustore = 0; /* XXX */
+	qae->qae_effect_ustore_size = USTORE_SIZE;
+
+	slice = &qae->qae_slices[qae->qae_num_slices];
+
+	slice->qas_image = qui;
+	slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned;
+
+	nregions = qui->qui_image->ui_num_page_regions;
+	npages = qui->qui_image->ui_num_pages;
+
+	if (nregions > nitems(slice->qas_regions))
+		return ENOENT;
+	if (npages > nitems(slice->qas_pages))
+		return ENOENT;
+
+	for (i = 0; i < nregions; i++) {
+		STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages);
+	}
+	for (i = 0; i < npages; i++) {
+		struct qat_ae_page *page = &slice->qas_pages[i];
+		int region;
+
+		page->qap_page = &qui->qui_pages[i];
+		region = page->qap_page->qup_page_region;
+		if (region >= nregions)
+			return EINVAL;
+
+		page->qap_region = &slice->qas_regions[region];
+	}
+
+	qae->qae_num_slices++;
+
+	return 0;
+}
+
+static int
+qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae)
+{
+	struct uof_image *image;
+	struct qat_ae *qae = &(QAT_AE(sc, ae));
+	int s;
+	u_char nn_mode;
+
+	for (s = 0; s < qae->qae_num_slices; s++) {
+		if (qae->qae_slices[s].qas_image == NULL)
+			continue;
+
+		image = qae->qae_slices[s].qas_image->qui_image;
+		qat_ae_write_ctx_mode(sc, ae,
+		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE));
+
+		nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE);
+		if (nn_mode != AE_MODE_NN_MODE_DONTCARE)
+			qat_ae_write_nn_mode(sc, ae, nn_mode);
+
+		qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0,
+		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0));
+		qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1,
+		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1));
+
+		qat_ae_write_shared_cs_mode(sc, ae,
+		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE));
+		qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size,
+		    __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED),
+		    qae->qae_reloc_ustore_dram);
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_uof_init(struct qat_softc *sc)
+{
+	int ae, i, error;
+	uint32_t mask;
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		struct qat_ae *qae;
+
+		if (!(mask & 1))
+			continue;
+
+		qae = &(QAT_AE(sc, ae));
+
+		for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
+			if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned &
+			    (1 << ae)) == 0)
+				continue;
+
+			error = qat_aefw_uof_assign_image(sc, qae,
+			    &sc->sc_aefw_uof.qafu_imgs[i]);
+			if (error)
+				return error;
+		}
+
+		/* XXX UcLo_initNumUwordUsed */
+
+		qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */
+
+		error = qat_aefw_uof_init_ae(sc, ae);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
+int
+qat_aefw_load(struct qat_softc *sc)
+{
+	int error;
+
+	error = qat_aefw_load_mof(sc);
+	if (error)
+		return error;
+
+	error = qat_aefw_load_mmp(sc);
+	if (error)
+		return error;
+
+	error = qat_aefw_mof_parse(sc);
+	if (error) {
+		device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error);
+		return error;
+	}
+
+	if (sc->sc_hw.qhw_fw_auth) {
+		error = qat_aefw_suof_parse(sc);
+		if (error) {
+			device_printf(sc->sc_dev, "couldn't parse suof: %d\n",
+			    error);
+			return error;
+		}
+
+		error = qat_aefw_suof_write(sc);
+		if (error) {
+			device_printf(sc->sc_dev,
+			    "could not write firmware: %d\n", error);
+			return error;
+		}
+
+	} else {
+		error = qat_aefw_uof_parse(sc);
+		if (error) {
+			device_printf(sc->sc_dev, "couldn't parse uof: %d\n",
+			    error);
+			return error;
+		}
+
+		error = qat_aefw_uof_init(sc);
+		if (error) {
+			device_printf(sc->sc_dev,
+			    "couldn't init for aefw: %d\n", error);
+			return error;
+		}
+
+		error = qat_aefw_uof_write(sc);
+		if (error) {
+			device_printf(sc->sc_dev,
+			    "Could not write firmware: %d\n", error);
+			return error;
+		}
+	}
+
+	return 0;
+}
+
+void
+qat_aefw_unload(struct qat_softc *sc)
+{
+	qat_aefw_unload_mmp(sc);
+	qat_aefw_unload_mof(sc);
+}
+
+int
+qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask)
+{
+	uint32_t fcu;
+	int retry = 0;
+
+	if (sc->sc_hw.qhw_fw_auth) {
+		qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START);
+		do {
+			DELAY(FW_AUTH_WAIT_PERIOD * 1000);
+			fcu = qat_cap_global_read_4(sc, FCU_STATUS);
+			if (fcu & FCU_STATUS_DONE)
+				return 0;
+		} while (retry++ < FW_AUTH_MAX_RETRY);
+
+		device_printf(sc->sc_dev,
+		    "firmware start timeout: status %08x\n", fcu);
+		return EINVAL;
+	} else {
+		qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX,
+		    CTX_WAKEUP_EVENTS_INDIRECT,
+		    CTX_WAKEUP_EVENTS_INDIRECT_SLEEP);
+		qat_ae_enable_ctx(sc, ae, ctx_mask);
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim)
+{
+	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
+	struct qat_ae_batch_init_list *qabi_list;
+	struct uof_mem_val_attr *memattr;
+	size_t *curinit;
+	u_long ael;
+	int i;
+	const char *sym;
+	char *ep;
+
+	memattr = (struct uof_mem_val_attr *)(uim + 1);
+
+	switch (uim->uim_region) {
+	case LMEM_REGION:
+		if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) {
+			device_printf(sc->sc_dev,
+			    "Invalid lmem addr or bytes\n");
+			return ENOBUFS;
+		}
+		if (uim->uim_scope != UOF_SCOPE_LOCAL)
+			return EINVAL;
+		sym = qat_aefw_uof_string(sc, uim->uim_sym_name);
+		ael = strtoul(sym, &ep, 10);
+		if (ep == sym || ael > MAX_AE)
+			return EINVAL;
+		if ((sc->sc_ae_mask & (1 << ael)) == 0)
+			return 0; /* ae is fused out */
+
+		curinit = &qafu->qafu_num_lm_init[ael];
+		qabi_list = &qafu->qafu_lm_init[ael];
+
+		for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) {
+			struct qat_ae_batch_init *qabi;
+
+			qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init));
+			if (*curinit == 0)
+				STAILQ_INIT(qabi_list);
+			STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next);
+
+			qabi->qabi_ae = (u_int)ael;
+			qabi->qabi_addr =
+			    uim->uim_addr + memattr->umva_byte_offset;
+			qabi->qabi_value = &memattr->umva_value;
+			qabi->qabi_size = 4;
+			qafu->qafu_num_lm_init_inst[ael] +=
+			    qat_ae_get_inst_num(qabi->qabi_size);
+			(*curinit)++;
+			if (*curinit >= MAX_LMEM_REG) {
+				device_printf(sc->sc_dev,
+				    "Invalid lmem val attr\n");
+				return ENOBUFS;
+			}
+		}
+		break;
+	case SRAM_REGION:
+	case DRAM_REGION:
+	case DRAM1_REGION:
+	case SCRATCH_REGION:
+	case UMEM_REGION:
+		/* XXX */
+		/* fallthrough */
+	default:
+		device_printf(sc->sc_dev,
+		    "unsupported memory region to init: %d\n",
+		    uim->uim_region);
+		return ENOTSUP;
+	}
+
+	return 0;
+}
+
+static void
+qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae)
+{
+	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
+	struct qat_ae_batch_init *qabi;
+
+	while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) {
+		STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next);
+		qat_free_mem(qabi);
+	}
+
+	qafu->qafu_num_lm_init[ae] = 0;
+	qafu->qafu_num_lm_init_inst[ae] = 0;
+}
+
+static int
+qat_aefw_init_ustore(struct qat_softc *sc)
+{
+	uint64_t *fill;
+	uint32_t dont_init;
+	int a, i, p;
+	int error = 0;
+	int usz, end, start;
+	u_char ae, nae;
+
+	fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t));
+
+	for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) {
+		struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a];
+		struct uof_image *ui = qui->qui_image;
+
+		for (i = 0; i < MAX_USTORE; i++)
+			memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t));
+		/*
+		 * Compute do_not_init value as a value that will not be equal
+		 * to fill data when cast to an int
+		 */
+		dont_init = 0;
+		if (dont_init == (uint32_t)fill[0])
+			dont_init = 0xffffffff;
+
+		for (p = 0; p < ui->ui_num_pages; p++) {
+			struct qat_uof_page *qup = &qui->qui_pages[p];
+			if (!qup->qup_def_page)
+				continue;
+
+			for (i = qup->qup_beg_paddr;
+			    i < qup->qup_beg_paddr + qup->qup_num_micro_words;
+			    i++ ) {
+				fill[i] = (uint64_t)dont_init;
+			}
+		}
+
+		for (ae = 0; ae < sc->sc_ae_num; ae++) {
+			MPASS(ae < UOF_MAX_NUM_OF_AE);
+			if ((ui->ui_ae_assigned & (1 << ae)) == 0)
+				continue;
+
+			if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) {
+				qat_ae_get_shared_ustore_ae(ae, &nae);
+				if (ui->ui_ae_assigned & (1 << ae))
+					continue;
+			}
+			usz = QAT_AE(sc, ae).qae_effect_ustore_size;
+
+			/* initialize the areas not going to be overwritten */
+			end = -1;
+			do {
+				/* find next uword that needs to be initialized */
+				for (start = end + 1; start < usz; start++) {
+					if ((uint32_t)fill[start] != dont_init)
+						break;
+				}
+				/* see if there are no more such uwords */
+				if (start >= usz)
+					break;
+				for (end = start + 1; end < usz; end++) {
+					if ((uint32_t)fill[end] == dont_init)
+						break;
+				}
+				if (QAT_AE(sc, ae).qae_shareable_ustore) {
+					error = ENOTSUP; /* XXX */
+					goto out;
+				} else {
+					error = qat_ae_ucode_write(sc, ae,
+					    start, end - start, &fill[start]);
+					if (error) {
+						goto out;
+					}
+				}
+
+			} while (end < usz);
+		}
+	}
+
+out:
+	qat_free_mem(fill);
+	return error;
+}
+
+static int
+qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask,
+    enum aereg_type regtype, u_short regaddr, u_int value)
+{
+	int error = 0;
+	u_char ctx;
+
+	switch (regtype) {
+	case AEREG_GPA_REL:
+	case AEREG_GPB_REL:
+	case AEREG_SR_REL:
+	case AEREG_SR_RD_REL:
+	case AEREG_SR_WR_REL:
+	case AEREG_DR_REL:
+	case AEREG_DR_RD_REL:
+	case AEREG_DR_WR_REL:
+	case AEREG_NEIGH_REL:
+		/* init for all valid ctx */
+		for (ctx = 0; ctx < MAX_AE_CTX; ctx++) {
+			if ((ctx_mask & (1 << ctx)) == 0)
+				continue;
+			error = qat_aereg_rel_data_write(sc, ae, ctx, regtype,
+			    regaddr, value);
+		}
+		break;
+	case AEREG_GPA_ABS:
+	case AEREG_GPB_ABS:
+	case AEREG_SR_ABS:
+	case AEREG_SR_RD_ABS:
+	case AEREG_SR_WR_ABS:
+	case AEREG_DR_ABS:
+	case AEREG_DR_RD_ABS:
+	case AEREG_DR_WR_ABS:
+		error = qat_aereg_abs_data_write(sc, ae, regtype,
+		    regaddr, value);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+
+	return error;
+}
+
+static int
+qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae,
+    struct qat_uof_image *qui)
+{
+	u_int i, expres;
+	u_char ctx_mask;
+
+	for (i = 0; i < qui->qui_num_init_reg_sym; i++) {
+		struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i];
+
+		if (uirs->uirs_value_type == EXPR_VAL) {
+			/* XXX */
+			device_printf(sc->sc_dev,
+			    "does not support initializing EXPR_VAL\n");
+			return ENOTSUP;
+		} else {
+			expres = uirs->uirs_value;
+		}
+
+		switch (uirs->uirs_init_type) {
+		case INIT_REG:
+			if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
+			    AE_MODE_CTX_MODE) == MAX_AE_CTX) {
+				ctx_mask = 0xff; /* 8-ctx mode */
+			} else {
+				ctx_mask = 0x55; /* 4-ctx mode */
+			}
+			qat_aefw_init_reg(sc, ae, ctx_mask,
+			    (enum aereg_type)uirs->uirs_reg_type,
+			    (u_short)uirs->uirs_addr_offset, expres);
+			break;
+		case INIT_REG_CTX:
+			if (__SHIFTOUT(qui->qui_image->ui_ae_mode,
+			    AE_MODE_CTX_MODE) == MAX_AE_CTX) {
+				ctx_mask = 0xff; /* 8-ctx mode */
+			} else {
+				ctx_mask = 0x55; /* 4-ctx mode */
+			}
+			if (((1 << uirs->uirs_ctx) & ctx_mask) == 0)
+				return EINVAL;
+			qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx,
+			    (enum aereg_type)uirs->uirs_reg_type,
+			    (u_short)uirs->uirs_addr_offset, expres);
+			break;
+		case INIT_EXPR:
+		case INIT_EXPR_ENDIAN_SWAP:
+		default:
+			device_printf(sc->sc_dev,
+			    "does not support initializing init_type %d\n",
+			    uirs->uirs_init_type);
+			return ENOTSUP;
+		}
+	}
+
+	return 0;
+}
+
+static int
+qat_aefw_init_memory(struct qat_softc *sc)
+{
+	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
+	size_t uimsz, initmemsz = qafu->qafu_init_mem_size;
+	struct uof_init_mem *uim;
+	int error, i;
+	u_char ae;
+
+	uim = qafu->qafu_init_mem;
+	for (i = 0; i < qafu->qafu_num_init_mem; i++) {
+		uimsz = sizeof(struct uof_init_mem) +
+		    sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr;
+		if (uimsz > initmemsz) {
+			device_printf(sc->sc_dev,
+			    "invalid uof_init_mem or uof_mem_val_attr size\n");
+			return EINVAL;
+		}
+
+		if (uim->uim_num_bytes > 0) {
+			error = qat_aefw_init_memory_one(sc, uim);
+			if (error) {
+				device_printf(sc->sc_dev,
+				    "Could not init ae memory: %d\n", error);
+				return error;
+			}
+		}
+		uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz);
+		initmemsz -= uimsz;
+	}
+
+	/* run Batch put LM API */
+	for (ae = 0; ae < MAX_AE; ae++) {
+		error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae],
+		    qafu->qafu_num_lm_init_inst[ae]);
+		if (error)
+			device_printf(sc->sc_dev, "Could not put lm\n");
+
+		qat_aefw_free_lm_init(sc, ae);
+	}
+
+	error = qat_aefw_init_ustore(sc);
+
+	/* XXX run Batch put LM API */
+
+	return error;
+}
+
+static int
+qat_aefw_init_globals(struct qat_softc *sc)
+{
+	struct qat_aefw_uof *qafu = &sc->sc_aefw_uof;
+	int error, i, p, s;
+	u_char ae;
+
+	/* initialize the memory segments */
+	if (qafu->qafu_num_init_mem > 0) {
+		error = qat_aefw_init_memory(sc);
+		if (error)
+			return error;
+	} else {
+		error = qat_aefw_init_ustore(sc);
+		if (error)
+			return error;
+	}
+
+	/* XXX bind import variables with ivd values */
+
+	/* XXX bind the uC global variables
+	 * local variables will done on-the-fly */
+	for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
+		for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) {
+			struct qat_uof_page *qup =
+			    &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p];
+			if (qup->qup_num_uw_blocks &&
+			    (qup->qup_num_uc_var || qup->qup_num_imp_var)) {
+				device_printf(sc->sc_dev,
+				    "not support uC global variables\n");
+				return ENOTSUP;
+			}
+		}
+	}
+
+	for (ae = 0; ae < sc->sc_ae_num; ae++) {
+		struct qat_ae *qae = &(QAT_AE(sc, ae));
+
+		for (s = 0; s < qae->qae_num_slices; s++) {
+			struct qat_ae_slice *qas = &qae->qae_slices[s];
+
+			if (qas->qas_image == NULL)
+				continue;
+
+			error =
+			    qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image);
+			if (error)
+				return error;
+		}
+	}
+
+	return 0;
+}
+
+static uint64_t
+qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup,
+    u_int addr)
+{
+	uint64_t uinst = 0;
+	u_int i;
+
+	/* find the block */
+	for (i = 0; i < qup->qup_num_uw_blocks; i++) {
+		struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i];
+
+		if ((addr >= quub->quub_start_addr) &&
+		    (addr <= (quub->quub_start_addr +
+		    (quub->quub_num_words - 1)))) {
+			/* unpack n bytes and assigned to the 64-bit uword value.
+			note: the microwords are stored as packed bytes.
+			*/
+			addr -= quub->quub_start_addr;
+			addr *= AEV2_PACKED_UWORD_BYTES;
+			memcpy(&uinst,
+			    (void *)((uintptr_t)quub->quub_micro_words + addr),
+			    AEV2_PACKED_UWORD_BYTES);
+			uinst = uinst & UWORD_MASK;
+
+			return uinst;
+		}
+	}
+
+	return INVLD_UWORD;
+}
+
+static int
+qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup)
+{
+	struct qat_ae *qae = &(QAT_AE(sc, ae));
+	uint64_t fill, *ucode_cpybuf;
+	u_int error, i, upaddr, uraddr, ninst, cpylen;
+
+	if (qup->qup_num_uc_var || qup->qup_num_neigh_reg ||
+	    qup->qup_num_imp_var || qup->qup_num_imp_expr) {
+		device_printf(sc->sc_dev,
+		    "does not support fixup locals\n");
+		return ENOTSUP;
+	}
+
+	ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t));
+
+	/* XXX get fill-pattern from an image -- they are all the same */
+	memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern,
+	    sizeof(uint64_t));
+
+	upaddr = qup->qup_beg_paddr;
+	uraddr = 0;
+	ninst = qup->qup_num_micro_words;
+	while (ninst > 0) {
+		cpylen = min(ninst, UWORD_CPYBUF_SIZE);
+
+		/* load the buffer */
+		for (i = 0; i < cpylen; i++) {
+			/* keep below code structure in case there are
+			 * different handling for shared secnarios */
+			if (!qae->qae_shareable_ustore) {
+				/* qat_aefw_get_uof_inst() takes an address that
+				 * is relative to the start of the page.
+				 * So we don't need to add in the physical
+				 * offset of the page. */
+				if (qup->qup_page_region != 0) {
+					/* XXX */
+					device_printf(sc->sc_dev,
+					    "region != 0 is not supported\n");
+					qat_free_mem(ucode_cpybuf);
+					return ENOTSUP;
+				} else {
+					/* for mixing case, it should take
+					 * physical address */
+					ucode_cpybuf[i] = qat_aefw_get_uof_inst(
+					    sc, qup, upaddr + i);
+					if (ucode_cpybuf[i] == INVLD_UWORD) {
+					    /* fill hole in the uof */
+					    ucode_cpybuf[i] = fill;
+					}
+				}
+			} else {
+				/* XXX */
+				qat_free_mem(ucode_cpybuf);
+				return ENOTSUP;
+			}
+		}
+
+		/* copy the buffer to ustore */
+		if (!qae->qae_shareable_ustore) {
+			error = qat_ae_ucode_write(sc, ae, upaddr, cpylen,
+			    ucode_cpybuf);
+			if (error)
+				return error;
+		} else {
+			/* XXX */
+			qat_free_mem(ucode_cpybuf);
+			return ENOTSUP;
+		}
+		upaddr += cpylen;
+		uraddr += cpylen;
+		ninst -= cpylen;
+	}
+
+	qat_free_mem(ucode_cpybuf);
+
+	return 0;
+}
+
+static int
+qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui)
+{
+	struct uof_image *ui = qui->qui_image;
+	struct qat_ae_page *qap;
+	u_int s, p, c;
+	int error;
+	u_char ae, ctx_mask;
+
+	if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX)
+		ctx_mask = 0xff; /* 8-ctx mode */
+	else
+		ctx_mask = 0x55; /* 4-ctx mode */
+
+	/* load the default page and set assigned CTX PC
+	 * to the entrypoint address */
+	for (ae = 0; ae < sc->sc_ae_num; ae++) {
+		struct qat_ae *qae = &(QAT_AE(sc, ae));
+		struct qat_ae_slice *qas;
+		u_int metadata;
+
+		MPASS(ae < UOF_MAX_NUM_OF_AE);
+
+		if ((ui->ui_ae_assigned & (1 << ae)) == 0)
+			continue;
+
+		/* find the slice to which this image is assigned */
+		for (s = 0; s < qae->qae_num_slices; s++) {
+			qas = &qae->qae_slices[s];
+			if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask)
+				break;
+		}
+		if (s >= qae->qae_num_slices)
+			continue;
+
+		qas = &qae->qae_slices[s];
+
+		for (p = 0; p < ui->ui_num_pages; p++) {
+			qap = &qas->qas_pages[p];
+
+			/* Only load pages loaded by default */
+			if (!qap->qap_page->qup_def_page)
+				continue;
+
+			error = qat_aefw_do_pagein(sc, ae, qap->qap_page);
+			if (error)
+				return error;
+		}
+
+		metadata = qas->qas_image->qui_image->ui_app_metadata;
+		if (metadata != 0xffffffff && bootverbose) {
+			device_printf(sc->sc_dev,
+			    "loaded firmware: %s\n",
+			    qat_aefw_uof_string(sc, metadata));
+		}
+
+		/* Assume starting page is page 0 */
+		qap = &qas->qas_pages[0];
+		for (c = 0; c < MAX_AE_CTX; c++) {
+			if (ctx_mask & (1 << c))
+				qas->qas_cur_pages[c] = qap;
+			else
+				qas->qas_cur_pages[c] = NULL;
+		}
+
+		/* set the live context */
+		qae->qae_live_ctx_mask = ui->ui_ctx_assigned;
+
+		/* set context PC to the image entrypoint address */
+		error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned,
+		    ui->ui_entry_address);
+		if (error)
+			return error;
+	}
+
+	/* XXX store the checksum for convenience */
+
+	return 0;
+}
+
+static int
+qat_aefw_uof_write(struct qat_softc *sc)
+{
+	int error = 0;
+	int i;
+
+	error = qat_aefw_init_globals(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Could not initialize globals\n");
+		return error;
+	}
+
+	for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) {
+		error = qat_aefw_uof_write_one(sc,
+		    &sc->sc_aefw_uof.qafu_imgs[i]);
+		if (error)
+			break;
+	}
+
+	/* XXX UcLo_computeFreeUstore */
+
+	return error;
+}
diff --git a/sys/dev/qat/qat_aevar.h b/sys/dev/qat/qat_aevar.h
new file mode 100644
index 000000000000..4aea6a5f8e99
--- /dev/null
+++ b/sys/dev/qat/qat_aevar.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_aevar.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_AEVAR_H_
+#define _DEV_PCI_QAT_AEVAR_H_
+
+int		qat_ae_init(struct qat_softc *);
+int		qat_ae_start(struct qat_softc *);
+void		qat_ae_cluster_intr(void *);
+
+int		qat_aefw_load(struct qat_softc *);
+void		qat_aefw_unload(struct qat_softc *);
+int		qat_aefw_start(struct qat_softc *, u_char, u_int);
+
+#endif
diff --git a/sys/dev/qat/qat_c2xxx.c b/sys/dev/qat/qat_c2xxx.c
new file mode 100644
index 000000000000..60ddafe07576
--- /dev/null
+++ b/sys/dev/qat/qat_c2xxx.c
@@ -0,0 +1,217 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_c2xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2013 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_c2xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qat_hw15reg.h"
+#include "qat_c2xxxreg.h"
+#include "qatvar.h"
+#include "qat_hw15var.h"
+
+static uint32_t
+qat_c2xxx_get_accel_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+
+	return ((~fusectl) & ACCEL_MASK_C2XXX);
+}
+
+static uint32_t
+qat_c2xxx_get_ae_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	if (fusectl & (
+	    FUSECTL_C2XXX_PKE_DISABLE |
+	    FUSECTL_C2XXX_ATH_DISABLE |
+	    FUSECTL_C2XXX_CPH_DISABLE)) {
+		return 0;
+	} else {
+		if ((~fusectl & AE_MASK_C2XXX) == 0x3) {
+			/*
+			 * With both AEs enabled we get spurious completions on
+			 * ETR rings.  Work around that for now by simply
+			 * disabling the second AE.
+			 */
+			device_printf(sc->sc_dev, "disabling second AE\n");
+			fusectl |= 0x2;
+		}
+		return ((~fusectl) & AE_MASK_C2XXX);
+	}
+}
+
+static enum qat_sku
+qat_c2xxx_get_sku(struct qat_softc *sc)
+{
+	uint32_t fusectl;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+
+	switch (sc->sc_ae_num) {
+	case 1:
+		if (fusectl & FUSECTL_C2XXX_LOW_SKU)
+			return QAT_SKU_3;
+		else if (fusectl & FUSECTL_C2XXX_MID_SKU)
+			return QAT_SKU_2;
+		break;
+	case MAX_AE_C2XXX:
+		return QAT_SKU_1;
+	}
+
+	return QAT_SKU_UNKNOWN;
+}
+
+static uint32_t
+qat_c2xxx_get_accel_cap(struct qat_softc *sc)
+{
+	return QAT_ACCEL_CAP_CRYPTO_SYMMETRIC |
+	    QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC |
+	    QAT_ACCEL_CAP_CIPHER |
+	    QAT_ACCEL_CAP_AUTHENTICATION;
+}
+
+static const char *
+qat_c2xxx_get_fw_uof_name(struct qat_softc *sc)
+{
+	if (sc->sc_rev < QAT_REVID_C2XXX_B0)
+		return AE_FW_UOF_NAME_C2XXX_A0;
+
+	/* QAT_REVID_C2XXX_B0 and QAT_REVID_C2XXX_C0 */
+	return AE_FW_UOF_NAME_C2XXX_B0;
+}
+
+static void
+qat_c2xxx_enable_intr(struct qat_softc *sc)
+{
+
+	qat_misc_write_4(sc, EP_SMIA_C2XXX, EP_SMIA_MASK_C2XXX);
+}
+
+static void
+qat_c2xxx_init_etr_intr(struct qat_softc *sc, int bank)
+{
+	/*
+	 * For now, all rings within the bank are setup such that the generation
+	 * of flag interrupts will be triggered when ring leaves the empty
+	 * state. Note that in order for the ring interrupt to generate an IRQ
+	 * the interrupt must also be enabled for the ring.
+	 */
+	qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL,
+	    ETR_INT_SRCSEL_MASK_0_C2XXX);
+	qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL_2,
+	    ETR_INT_SRCSEL_MASK_X_C2XXX);
+}
+
+const struct qat_hw qat_hw_c2xxx = {
+	.qhw_sram_bar_id = BAR_SRAM_ID_C2XXX,
+	.qhw_misc_bar_id = BAR_PMISC_ID_C2XXX,
+	.qhw_etr_bar_id = BAR_ETR_ID_C2XXX,
+	.qhw_cap_global_offset = CAP_GLOBAL_OFFSET_C2XXX,
+	.qhw_ae_offset = AE_OFFSET_C2XXX,
+	.qhw_ae_local_offset = AE_LOCAL_OFFSET_C2XXX,
+	.qhw_etr_bundle_size = ETR_BUNDLE_SIZE_C2XXX,
+	.qhw_num_banks = ETR_MAX_BANKS_C2XXX,
+	.qhw_num_ap_banks = ETR_MAX_AP_BANKS_C2XXX,
+	.qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK,
+	.qhw_num_accel = MAX_ACCEL_C2XXX,
+	.qhw_num_engines = MAX_AE_C2XXX,
+	.qhw_tx_rx_gap = ETR_TX_RX_GAP_C2XXX,
+	.qhw_tx_rings_mask = ETR_TX_RINGS_MASK_C2XXX,
+	.qhw_msix_ae_vec_gap = MSIX_AE_VEC_GAP_C2XXX,
+	.qhw_fw_auth = false,
+	.qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW15,
+	.qhw_fw_resp_size = FW_REQ_DEFAULT_SZ_HW15,
+	.qhw_ring_asym_tx = 2,
+	.qhw_ring_asym_rx = 3,
+	.qhw_ring_sym_tx = 4,
+	.qhw_ring_sym_rx = 5,
+	.qhw_mof_fwname = AE_FW_MOF_NAME_C2XXX,
+	.qhw_mmp_fwname = AE_FW_MMP_NAME_C2XXX,
+	.qhw_prod_type = AE_FW_PROD_TYPE_C2XXX,
+	.qhw_get_accel_mask = qat_c2xxx_get_accel_mask,
+	.qhw_get_ae_mask = qat_c2xxx_get_ae_mask,
+	.qhw_get_sku = qat_c2xxx_get_sku,
+	.qhw_get_accel_cap = qat_c2xxx_get_accel_cap,
+	.qhw_get_fw_uof_name = qat_c2xxx_get_fw_uof_name,
+	.qhw_enable_intr = qat_c2xxx_enable_intr,
+	.qhw_init_etr_intr = qat_c2xxx_init_etr_intr,
+	.qhw_init_admin_comms = qat_adm_ring_init,
+	.qhw_send_admin_init = qat_adm_ring_send_init,
+	.qhw_crypto_setup_desc = qat_hw15_crypto_setup_desc,
+	.qhw_crypto_setup_req_params = qat_hw15_crypto_setup_req_params,
+	.qhw_crypto_opaque_offset =
+	    offsetof(struct fw_la_resp, comn_resp.opaque_data),
+};
diff --git a/sys/dev/qat/qat_c2xxxreg.h b/sys/dev/qat/qat_c2xxxreg.h
new file mode 100644
index 000000000000..3d5cae51138e
--- /dev/null
+++ b/sys/dev/qat/qat_c2xxxreg.h
@@ -0,0 +1,177 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_c2xxxreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2013 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_C2XXXREG_H_
+#define _DEV_PCI_QAT_C2XXXREG_H_
+
+/* PCI revision IDs */
+#define QAT_REVID_C2XXX_A0		0x00
+#define QAT_REVID_C2XXX_B0		0x02
+#define QAT_REVID_C2XXX_C0		0x03
+
+/* Max number of accelerators and engines */
+#define MAX_ACCEL_C2XXX			1
+#define MAX_AE_C2XXX			2
+
+/* PCIe BAR index */
+#define BAR_SRAM_ID_C2XXX		NO_PCI_REG
+#define BAR_PMISC_ID_C2XXX		0
+#define BAR_ETR_ID_C2XXX		1
+
+#define ACCEL_MASK_C2XXX		0x1
+#define AE_MASK_C2XXX			0x3
+
+#define MSIX_AE_VEC_GAP_C2XXX		8
+
+/* PCIe configuration space registers */
+/* PESRAM: 512K eSRAM */
+#define BAR_PESRAM_C2XXX		NO_PCI_REG
+#define BAR_PESRAM_SIZE_C2XXX		0
+
+/*
+ * PMISC: 16K CAP, 16K Scratch, 32K SSU(QATs),
+ *        32K AE CSRs and transfer registers, 8K CHAP/PMU,
+ *        4K EP CSRs, 4K MSI-X Tables
+ */
+#define BAR_PMISC_C2XXX			0x18
+#define BAR_PMISC_SIZE_C2XXX		0x20000	/* 128K */
+
+/* PETRINGCSR: 8K 16 bundles of ET Ring CSRs */
+#define BAR_PETRINGCSR_C2XXX		0x20
+#define BAR_PETRINGCSR_SIZE_C2XXX	0x4000	/* 16K */
+
+/* Fuse Control */
+#define FUSECTL_C2XXX_PKE_DISABLE	(1 << 6)
+#define FUSECTL_C2XXX_ATH_DISABLE	(1 << 5)
+#define FUSECTL_C2XXX_CPH_DISABLE	(1 << 4)
+#define FUSECTL_C2XXX_LOW_SKU		(1 << 3)
+#define FUSECTL_C2XXX_MID_SKU		(1 << 2)
+#define FUSECTL_C2XXX_AE1_DISABLE	(1 << 1)
+
+/* SINT: Signal Target Raw Interrupt Register */
+#define EP_SINTPF_C2XXX			0x1A024
+
+/* SMIA: Signal Target IA Mask Register */
+#define EP_SMIA_C2XXX				0x1A028
+#define EP_SMIA_BUNDLES_IRQ_MASK_C2XXX		0xFF
+#define EP_SMIA_AE_IRQ_MASK_C2XXX		0x10000
+#define EP_SMIA_MASK_C2XXX			\
+	(EP_SMIA_BUNDLES_IRQ_MASK_C2XXX | EP_SMIA_AE_IRQ_MASK_C2XXX)
+
+#define EP_RIMISCCTL_C2XXX		0x1A0C4
+#define EP_RIMISCCTL_MASK_C2XXX		0x40000000
+
+#define PFCGCIOSFPRIR_REG_C2XXX			0x2C0
+#define PFCGCIOSFPRIR_MASK_C2XXX		0XFFFF7FFF
+
+/* BAR sub-regions */
+#define PESRAM_BAR_C2XXX		NO_PCI_REG
+#define PESRAM_OFFSET_C2XXX		0x0
+#define PESRAM_SIZE_C2XXX		0x0
+#define CAP_GLOBAL_BAR_C2XXX		BAR_PMISC_C2XXX
+#define CAP_GLOBAL_OFFSET_C2XXX		0x00000
+#define CAP_GLOBAL_SIZE_C2XXX		0x04000
+#define CAP_HASH_OFFSET			0x900
+#define SCRATCH_BAR_C2XXX		NO_PCI_REG
+#define SCRATCH_OFFSET_C2XXX		NO_REG_OFFSET
+#define SCRATCH_SIZE_C2XXX		0x0
+#define SSU_BAR_C2XXX			BAR_PMISC_C2XXX
+#define SSU_OFFSET_C2XXX		0x08000
+#define SSU_SIZE_C2XXX			0x08000
+#define AE_BAR_C2XXX			BAR_PMISC_C2XXX
+#define AE_OFFSET_C2XXX			0x10000
+#define AE_LOCAL_OFFSET_C2XXX		0x10800
+#define PMU_BAR_C2XXX			NO_PCI_REG
+#define PMU_OFFSET_C2XXX		NO_REG_OFFSET
+#define PMU_SIZE_C2XXX			0x0
+#define EP_BAR_C2XXX			BAR_PMISC_C2XXX
+#define EP_OFFSET_C2XXX			0x1A000
+#define EP_SIZE_C2XXX			0x01000
+#define MSIX_TAB_BAR_C2XXX		NO_PCI_REG	/* mapped by pci(9) */
+#define MSIX_TAB_OFFSET_C2XXX		0x1B000
+#define MSIX_TAB_SIZE_C2XXX		0x01000
+#define PETRINGCSR_BAR_C2XXX		BAR_PETRINGCSR_C2XXX
+#define PETRINGCSR_OFFSET_C2XXX		0x0
+#define PETRINGCSR_SIZE_C2XXX		0x0	/* use size of BAR */
+
+/* ETR */
+#define ETR_MAX_BANKS_C2XXX		8
+#define ETR_MAX_ET_RINGS_C2XXX		\
+	(ETR_MAX_BANKS_C2XXX * ETR_MAX_RINGS_PER_BANK_C2XXX)
+#define ETR_MAX_AP_BANKS_C2XXX		4
+
+#define ETR_TX_RX_GAP_C2XXX		1
+#define ETR_TX_RINGS_MASK_C2XXX		0x51
+
+#define ETR_BUNDLE_SIZE_C2XXX		0x0200
+
+/* Initial bank Interrupt Source mask */
+#define ETR_INT_SRCSEL_MASK_0_C2XXX	0x4444444CUL
+#define ETR_INT_SRCSEL_MASK_X_C2XXX	0x44444444UL
+
+/* AE firmware */
+#define AE_FW_PROD_TYPE_C2XXX			0x00800000
+#define AE_FW_MOF_NAME_C2XXX		"mof_firmware_c2xxx"
+#define AE_FW_MMP_NAME_C2XXX		"mmp_firmware_c2xxx"
+#define AE_FW_UOF_NAME_C2XXX_A0		"icp_qat_nae.uof"
+#define AE_FW_UOF_NAME_C2XXX_B0		"icp_qat_nae_b0.uof"
+
+#endif
diff --git a/sys/dev/qat/qat_c3xxx.c b/sys/dev/qat/qat_c3xxx.c
new file mode 100644
index 000000000000..83d39da7d0af
--- /dev/null
+++ b/sys/dev/qat/qat_c3xxx.c
@@ -0,0 +1,298 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_c3xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_c3xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qat_hw17reg.h"
+#include "qat_c3xxxreg.h"
+#include "qatvar.h"
+#include "qat_hw17var.h"
+
+static uint32_t
+qat_c3xxx_get_accel_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, strap;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C3XXX, 4);
+
+	return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_C3XXX) &
+	    ACCEL_MASK_C3XXX);
+}
+
+static uint32_t
+qat_c3xxx_get_ae_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, me_strap, me_disable, ssms_disabled;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	me_strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C3XXX, 4);
+
+	/* If SSMs are disabled, then disable the corresponding MEs */
+	ssms_disabled = (~qat_c3xxx_get_accel_mask(sc)) & ACCEL_MASK_C3XXX;
+	me_disable = 0x3;
+	while (ssms_disabled) {
+		if (ssms_disabled & 1)
+			me_strap |= me_disable;
+		ssms_disabled >>= 1;
+		me_disable <<= 2;
+	}
+
+	return (~(fusectl | me_strap)) & AE_MASK_C3XXX;
+}
+
+static enum qat_sku
+qat_c3xxx_get_sku(struct qat_softc *sc)
+{
+	switch (sc->sc_ae_num) {
+	case MAX_AE_C3XXX:
+		return QAT_SKU_4;
+	}
+
+	return QAT_SKU_UNKNOWN;
+}
+
+static uint32_t
+qat_c3xxx_get_accel_cap(struct qat_softc *sc)
+{
+	uint32_t cap, legfuse, strap;
+
+	legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C3XXX, 4);
+
+	cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC +
+		QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC +
+		QAT_ACCEL_CAP_CIPHER +
+		QAT_ACCEL_CAP_AUTHENTICATION +
+		QAT_ACCEL_CAP_COMPRESSION +
+		QAT_ACCEL_CAP_ZUC +
+		QAT_ACCEL_CAP_SHA3;
+
+	if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) {
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC;
+		cap &= ~QAT_ACCEL_CAP_CIPHER;
+	}
+	if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE)
+		cap &= ~QAT_ACCEL_CAP_AUTHENTICATION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE)
+		cap &= ~QAT_ACCEL_CAP_ZUC;
+
+	if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_PKE_C3XXX)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_CY_C3XXX)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+
+	return cap;
+}
+
+static const char *
+qat_c3xxx_get_fw_uof_name(struct qat_softc *sc)
+{
+
+	return AE_FW_UOF_NAME_C3XXX;
+}
+
+static void
+qat_c3xxx_enable_intr(struct qat_softc *sc)
+{
+
+	/* Enable bundle and misc interrupts */
+	qat_misc_write_4(sc, SMIAPF0_C3XXX, SMIA0_MASK_C3XXX);
+	qat_misc_write_4(sc, SMIAPF1_C3XXX, SMIA1_MASK_C3XXX);
+}
+
+/* Worker thread to service arbiter mappings */
+static uint32_t thrd_to_arb_map[] = {
+	0x12222AAA, 0x11222AAA, 0x12222AAA,
+	0x11222AAA, 0x12222AAA, 0x11222AAA
+};
+
+static void
+qat_c3xxx_get_arb_mapping(struct qat_softc *sc, const uint32_t **arb_map_config)
+{
+	int i;
+
+	for (i = 1; i < MAX_AE_C3XXX; i++) {
+		if ((~sc->sc_ae_mask) & (1 << i))
+			thrd_to_arb_map[i] = 0;
+	}
+	*arb_map_config = thrd_to_arb_map;
+}
+
+static void
+qat_c3xxx_enable_error_interrupts(struct qat_softc *sc)
+{
+	qat_misc_write_4(sc, ERRMSK0, ERRMSK0_CERR_C3XXX); /* ME0-ME3 */
+	qat_misc_write_4(sc, ERRMSK1, ERRMSK1_CERR_C3XXX); /* ME4-ME5 */
+	qat_misc_write_4(sc, ERRMSK5, ERRMSK5_CERR_C3XXX); /* SSM2 */
+
+	/* Reset everything except VFtoPF1_16. */
+	qat_misc_read_write_and_4(sc, ERRMSK3, VF2PF1_16_C3XXX);
+
+	/* RI CPP bus interface error detection and reporting. */
+	qat_misc_write_4(sc, RICPPINTCTL_C3XXX, RICPP_EN_C3XXX);
+
+	/* TI CPP bus interface error detection and reporting. */
+	qat_misc_write_4(sc, TICPPINTCTL_C3XXX, TICPP_EN_C3XXX);
+
+	/* Enable CFC Error interrupts and logging. */
+	qat_misc_write_4(sc, CPP_CFC_ERR_CTRL_C3XXX, CPP_CFC_UE_C3XXX);
+}
+
+static void
+qat_c3xxx_disable_error_interrupts(struct qat_softc *sc)
+{
+	/* ME0-ME3 */
+	qat_misc_write_4(sc, ERRMSK0, ERRMSK0_UERR_C3XXX | ERRMSK0_CERR_C3XXX);
+	/* ME4-ME5 */
+	qat_misc_write_4(sc, ERRMSK1, ERRMSK1_UERR_C3XXX | ERRMSK1_CERR_C3XXX);
+	/* CPP Push Pull, RI, TI, SSM0-SSM1, CFC */
+	qat_misc_write_4(sc, ERRMSK3, ERRMSK3_UERR_C3XXX);
+	/* SSM2 */
+	qat_misc_write_4(sc, ERRMSK5, ERRMSK5_UERR_C3XXX);
+}
+
+static void
+qat_c3xxx_enable_error_correction(struct qat_softc *sc)
+{
+	u_int i, mask;
+
+	/* Enable Accel Engine error detection & correction */
+	for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_C3XXX(i),
+		    ENABLE_AE_ECC_ERR_C3XXX);
+		qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_C3XXX(i),
+		    ENABLE_AE_ECC_PARITY_CORR_C3XXX);
+	}
+
+	/* Enable shared memory error detection & correction */
+	for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_C3XXX);
+		qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_C3XXX);
+		qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_C3XXX);
+	}
+
+	qat_c3xxx_enable_error_interrupts(sc);
+}
+
+const struct qat_hw qat_hw_c3xxx = {
+	.qhw_sram_bar_id = BAR_SRAM_ID_C3XXX,
+	.qhw_misc_bar_id = BAR_PMISC_ID_C3XXX,
+	.qhw_etr_bar_id = BAR_ETR_ID_C3XXX,
+	.qhw_cap_global_offset = CAP_GLOBAL_OFFSET_C3XXX,
+	.qhw_ae_offset = AE_OFFSET_C3XXX,
+	.qhw_ae_local_offset = AE_LOCAL_OFFSET_C3XXX,
+	.qhw_etr_bundle_size = ETR_BUNDLE_SIZE_C3XXX,
+	.qhw_num_banks = ETR_MAX_BANKS_C3XXX,
+	.qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK,
+	.qhw_num_accel = MAX_ACCEL_C3XXX,
+	.qhw_num_engines = MAX_AE_C3XXX,
+	.qhw_tx_rx_gap = ETR_TX_RX_GAP_C3XXX,
+	.qhw_tx_rings_mask = ETR_TX_RINGS_MASK_C3XXX,
+	.qhw_clock_per_sec = CLOCK_PER_SEC_C3XXX,
+	.qhw_fw_auth = true,
+	.qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17,
+	.qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17,
+	.qhw_ring_asym_tx = 0,
+	.qhw_ring_asym_rx = 8,
+	.qhw_ring_sym_tx = 2,
+	.qhw_ring_sym_rx = 10,
+	.qhw_mof_fwname = AE_FW_MOF_NAME_C3XXX,
+	.qhw_mmp_fwname = AE_FW_MMP_NAME_C3XXX,
+	.qhw_prod_type = AE_FW_PROD_TYPE_C3XXX,
+	.qhw_get_accel_mask = qat_c3xxx_get_accel_mask,
+	.qhw_get_ae_mask = qat_c3xxx_get_ae_mask,
+	.qhw_get_sku = qat_c3xxx_get_sku,
+	.qhw_get_accel_cap = qat_c3xxx_get_accel_cap,
+	.qhw_get_fw_uof_name = qat_c3xxx_get_fw_uof_name,
+	.qhw_enable_intr = qat_c3xxx_enable_intr,
+	.qhw_init_admin_comms = qat_adm_mailbox_init,
+	.qhw_send_admin_init = qat_adm_mailbox_send_init,
+	.qhw_init_arb = qat_arb_init,
+	.qhw_get_arb_mapping = qat_c3xxx_get_arb_mapping,
+	.qhw_enable_error_correction = qat_c3xxx_enable_error_correction,
+	.qhw_disable_error_interrupts = qat_c3xxx_disable_error_interrupts,
+	.qhw_set_ssm_wdtimer = qat_set_ssm_wdtimer,
+	.qhw_check_slice_hang = qat_check_slice_hang,
+	.qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc,
+	.qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params,
+	.qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data),
+};
diff --git a/sys/dev/qat/qat_c3xxxreg.h b/sys/dev/qat/qat_c3xxxreg.h
new file mode 100644
index 000000000000..9c4215bc0099
--- /dev/null
+++ b/sys/dev/qat/qat_c3xxxreg.h
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_c3xxxreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_C3XXXREG_H_
+#define _DEV_PCI_QAT_C3XXXREG_H_
+
+/* Max number of accelerators and engines */
+#define MAX_ACCEL_C3XXX			3
+#define MAX_AE_C3XXX			6
+
+/* PCIe BAR index */
+#define BAR_SRAM_ID_C3XXX		NO_PCI_REG
+#define BAR_PMISC_ID_C3XXX		0
+#define BAR_ETR_ID_C3XXX		1
+
+/* BAR PMISC sub-regions */
+#define AE_OFFSET_C3XXX			0x20000
+#define AE_LOCAL_OFFSET_C3XXX		0x20800
+#define CAP_GLOBAL_OFFSET_C3XXX		0x30000
+
+#define SOFTSTRAP_REG_C3XXX			0x2EC
+#define SOFTSTRAP_SS_POWERGATE_CY_C3XXX		__BIT(23)
+#define SOFTSTRAP_SS_POWERGATE_PKE_C3XXX	__BIT(24)
+
+#define ACCEL_REG_OFFSET_C3XXX		16
+#define ACCEL_MASK_C3XXX		0x7
+#define AE_MASK_C3XXX			0x3F
+
+#define SMIAPF0_C3XXX			0x3A028
+#define SMIAPF1_C3XXX			0x3A030
+#define SMIA0_MASK_C3XXX		0xFFFF
+#define SMIA1_MASK_C3XXX		0x1
+
+/* Error detection and correction */
+#define AE_CTX_ENABLES_C3XXX(i)		((i) * 0x1000 + 0x20818)
+#define AE_MISC_CONTROL_C3XXX(i)	((i) * 0x1000 + 0x20960)
+#define ENABLE_AE_ECC_ERR_C3XXX		__BIT(28)
+#define ENABLE_AE_ECC_PARITY_CORR_C3XXX	(__BIT(24) | __BIT(12))
+#define ERRSSMSH_EN_C3XXX		__BIT(3)
+/* BIT(2) enables the logging of push/pull data errors. */
+#define PPERR_EN_C3XXX			(__BIT(2))
+
+/* Mask for VF2PF interrupts */
+#define VF2PF1_16_C3XXX			(0xFFFF << 9)
+#define ERRSOU3_VF2PF_C3XXX(errsou3)	(((errsou3) & 0x01FFFE00) >> 9)
+#define ERRMSK3_VF2PF_C3XXX(vf_mask)	(((vf_mask) & 0xFFFF) << 9)
+
+/* Masks for correctable error interrupts. */
+#define ERRMSK0_CERR_C3XXX		(__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0))
+#define ERRMSK1_CERR_C3XXX		(__BIT(8) | __BIT(0))
+#define ERRMSK5_CERR_C3XXX		(0)
+
+/* Masks for uncorrectable error interrupts. */
+#define ERRMSK0_UERR_C3XXX		(__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1))
+#define ERRMSK1_UERR_C3XXX		(__BIT(9) | __BIT(1))
+#define ERRMSK3_UERR_C3XXX		(__BIT(6) | __BIT(5) | __BIT(4) | __BIT(3) | \
+					 __BIT(2) | __BIT(0))
+#define ERRMSK5_UERR_C3XXX		(__BIT(16))
+
+/* RI CPP control */
+#define RICPPINTCTL_C3XXX		(0x3A000 + 0x110)
+/*
+ * BIT(2) enables error detection and reporting on the RI Parity Error.
+ * BIT(1) enables error detection and reporting on the RI CPP Pull interface.
+ * BIT(0) enables error detection and reporting on the RI CPP Push interface.
+ */
+#define RICPP_EN_C3XXX			(__BIT(2) | __BIT(1) | __BIT(0))
+
+/* TI CPP control */
+#define TICPPINTCTL_C3XXX		(0x3A400 + 0x138)
+/*
+ * BIT(3) enables error detection and reporting on the ETR Parity Error.
+ * BIT(2) enables error detection and reporting on the TI Parity Error.
+ * BIT(1) enables error detection and reporting on the TI CPP Pull interface.
+ * BIT(0) enables error detection and reporting on the TI CPP Push interface.
+ */
+#define TICPP_EN_C3XXX		\
+	(__BIT(3) | __BIT(2) | __BIT(1) | __BIT(0))
+
+/* CFC Uncorrectable Errors */
+#define CPP_CFC_ERR_CTRL_C3XXX	(0x30000 + 0xC00)
+/*
+ * BIT(1) enables interrupt.
+ * BIT(0) enables detecting and logging of push/pull data errors.
+ */
+#define CPP_CFC_UE_C3XXX		(__BIT(1) | __BIT(0))
+
+#define SLICEPWRDOWN_C3XXX(i)	((i) * 0x4000 + 0x2C)
+/* Enabling PKE4-PKE0. */
+#define MMP_PWR_UP_MSK_C3XXX		\
+	(__BIT(20) | __BIT(19) | __BIT(18) | __BIT(17) | __BIT(16))
+
+/* CPM Uncorrectable Errors */
+#define INTMASKSSM_C3XXX(i)		((i) * 0x4000 + 0x0)
+/* Disabling interrupts for correctable errors. */
+#define INTMASKSSM_UERR_C3XXX	\
+	(__BIT(11) | __BIT(9) | __BIT(7) | __BIT(5) | __BIT(3) | __BIT(1))
+
+/* MMP */
+/* BIT(3) enables correction. */
+#define CERRSSMMMP_EN_C3XXX		(__BIT(3))
+
+/* BIT(3) enables logging. */
+#define UERRSSMMMP_EN_C3XXX		(__BIT(3))
+
+/* ETR */
+#define ETR_MAX_BANKS_C3XXX		16
+#define ETR_TX_RX_GAP_C3XXX		8
+#define ETR_TX_RINGS_MASK_C3XXX		0xFF
+#define ETR_BUNDLE_SIZE_C3XXX		0x1000
+
+/* AE firmware */
+#define AE_FW_PROD_TYPE_C3XXX		0x02000000
+#define AE_FW_MOF_NAME_C3XXX	"qat_c3xxx"
+#define AE_FW_MMP_NAME_C3XXX	"qat_c3xxx_mmp"
+#define AE_FW_UOF_NAME_C3XXX	"icp_qat_ae.suof"
+
+/* Clock frequency */
+#define CLOCK_PER_SEC_C3XXX		(685 * 1000000 / 16)
+
+#endif
diff --git a/sys/dev/qat/qat_c62x.c b/sys/dev/qat/qat_c62x.c
new file mode 100644
index 000000000000..826c68db9854
--- /dev/null
+++ b/sys/dev/qat/qat_c62x.c
@@ -0,0 +1,314 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_c62x.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_c62x.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qat_hw17reg.h"
+#include "qat_c62xreg.h"
+#include "qatvar.h"
+#include "qat_hw17var.h"
+
+static uint32_t
+qat_c62x_get_accel_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, strap;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C62X, 4);
+
+	return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_C62X) &
+	    ACCEL_MASK_C62X);
+}
+
+static uint32_t
+qat_c62x_get_ae_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, me_strap, me_disable, ssms_disabled;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	me_strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C62X, 4);
+
+	/* If SSMs are disabled, then disable the corresponding MEs */
+	ssms_disabled = (~qat_c62x_get_accel_mask(sc)) & ACCEL_MASK_C62X;
+	me_disable = 0x3;
+	while (ssms_disabled) {
+		if (ssms_disabled & 1)
+			me_strap |= me_disable;
+		ssms_disabled >>= 1;
+		me_disable <<= 2;
+	}
+
+	return (~(fusectl | me_strap)) & AE_MASK_C62X;
+}
+
+static enum qat_sku
+qat_c62x_get_sku(struct qat_softc *sc)
+{
+	switch (sc->sc_ae_num) {
+	case 8:
+		return QAT_SKU_2;
+	case MAX_AE_C62X:
+		return QAT_SKU_4;
+	}
+
+	return QAT_SKU_UNKNOWN;
+}
+
+static uint32_t
+qat_c62x_get_accel_cap(struct qat_softc *sc)
+{
+	uint32_t cap, legfuse, strap;
+
+	legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C62X, 4);
+
+	cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC +
+		QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC +
+		QAT_ACCEL_CAP_CIPHER +
+		QAT_ACCEL_CAP_AUTHENTICATION +
+		QAT_ACCEL_CAP_COMPRESSION +
+		QAT_ACCEL_CAP_ZUC +
+		QAT_ACCEL_CAP_SHA3;
+
+	if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) {
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC;
+		cap &= ~QAT_ACCEL_CAP_CIPHER;
+	}
+	if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE)
+		cap &= ~QAT_ACCEL_CAP_AUTHENTICATION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE)
+		cap &= ~QAT_ACCEL_CAP_ZUC;
+
+	if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_PKE_C62X)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_CY_C62X)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+
+	return cap;
+}
+
+static const char *
+qat_c62x_get_fw_uof_name(struct qat_softc *sc)
+{
+
+	return AE_FW_UOF_NAME_C62X;
+}
+
+static void
+qat_c62x_enable_intr(struct qat_softc *sc)
+{
+
+	/* Enable bundle and misc interrupts */
+	qat_misc_write_4(sc, SMIAPF0_C62X, SMIA0_MASK_C62X);
+	qat_misc_write_4(sc, SMIAPF1_C62X, SMIA1_MASK_C62X);
+}
+
+/* Worker thread to service arbiter mappings */
+static uint32_t thrd_to_arb_map[] = {
+	0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA,
+	0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA
+};
+
+static void
+qat_c62x_get_arb_mapping(struct qat_softc *sc, const uint32_t **arb_map_config)
+{
+	int i;
+
+	for (i = 1; i < MAX_AE_C62X; i++) {
+		if ((~sc->sc_ae_mask) & (1 << i))
+			thrd_to_arb_map[i] = 0;
+	}
+	*arb_map_config = thrd_to_arb_map;
+}
+
+static void
+qat_c62x_enable_error_interrupts(struct qat_softc *sc)
+{
+	qat_misc_write_4(sc, ERRMSK0, ERRMSK0_CERR_C62X); /* ME0-ME3 */
+	qat_misc_write_4(sc, ERRMSK1, ERRMSK1_CERR_C62X); /* ME4-ME7 */
+	qat_misc_write_4(sc, ERRMSK4, ERRMSK4_CERR_C62X); /* ME8-ME9 */
+	qat_misc_write_4(sc, ERRMSK5, ERRMSK5_CERR_C62X); /* SSM2-SSM4 */
+
+	/* Reset everything except VFtoPF1_16. */
+	qat_misc_read_write_and_4(sc, ERRMSK3, VF2PF1_16_C62X);
+	/* Disable Secure RAM correctable error interrupt */
+	qat_misc_read_write_or_4(sc, ERRMSK3, ERRMSK3_CERR_C62X);
+
+	/* RI CPP bus interface error detection and reporting. */
+	qat_misc_write_4(sc, RICPPINTCTL_C62X, RICPP_EN_C62X);
+
+	/* TI CPP bus interface error detection and reporting. */
+	qat_misc_write_4(sc, TICPPINTCTL_C62X, TICPP_EN_C62X);
+
+	/* Enable CFC Error interrupts and logging. */
+	qat_misc_write_4(sc, CPP_CFC_ERR_CTRL_C62X, CPP_CFC_UE_C62X);
+
+	/* Enable SecureRAM to fix and log Correctable errors */
+	qat_misc_write_4(sc, SECRAMCERR_C62X, SECRAM_CERR_C62X);
+
+	/* Enable SecureRAM Uncorrectable error interrupts and logging */
+	qat_misc_write_4(sc, SECRAMUERR, SECRAM_UERR_C62X);
+
+	/* Enable Push/Pull Misc Uncorrectable error interrupts and logging */
+	qat_misc_write_4(sc, CPPMEMTGTERR, TGT_UERR_C62X);
+}
+
+static void
+qat_c62x_disable_error_interrupts(struct qat_softc *sc)
+{
+	/* ME0-ME3 */
+	qat_misc_write_4(sc, ERRMSK0, ERRMSK0_UERR_C62X | ERRMSK0_CERR_C62X);
+	/* ME4-ME7 */
+	qat_misc_write_4(sc, ERRMSK1, ERRMSK1_UERR_C62X | ERRMSK1_CERR_C62X);
+	/* Secure RAM, CPP Push Pull, RI, TI, SSM0-SSM1, CFC */
+	qat_misc_write_4(sc, ERRMSK3, ERRMSK3_UERR_C62X | ERRMSK3_CERR_C62X);
+	/* ME8-ME9 */
+	qat_misc_write_4(sc, ERRMSK4, ERRMSK4_UERR_C62X | ERRMSK4_CERR_C62X);
+	/* SSM2-SSM4 */
+	qat_misc_write_4(sc, ERRMSK5, ERRMSK5_UERR_C62X | ERRMSK5_CERR_C62X);
+}
+
+static void
+qat_c62x_enable_error_correction(struct qat_softc *sc)
+{
+	u_int i, mask;
+
+	/* Enable Accel Engine error detection & correction */
+	for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_C62X(i),
+		    ENABLE_AE_ECC_ERR_C62X);
+		qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_C62X(i),
+		    ENABLE_AE_ECC_PARITY_CORR_C62X);
+	}
+
+	/* Enable shared memory error detection & correction */
+	for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_C62X);
+		qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_C62X);
+		qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_C62X);
+	}
+
+	qat_c62x_enable_error_interrupts(sc);
+}
+
+const struct qat_hw qat_hw_c62x = {
+	.qhw_sram_bar_id = BAR_SRAM_ID_C62X,
+	.qhw_misc_bar_id = BAR_PMISC_ID_C62X,
+	.qhw_etr_bar_id = BAR_ETR_ID_C62X,
+	.qhw_cap_global_offset = CAP_GLOBAL_OFFSET_C62X,
+	.qhw_ae_offset = AE_OFFSET_C62X,
+	.qhw_ae_local_offset = AE_LOCAL_OFFSET_C62X,
+	.qhw_etr_bundle_size = ETR_BUNDLE_SIZE_C62X,
+	.qhw_num_banks = ETR_MAX_BANKS_C62X,
+	.qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK,
+	.qhw_num_accel = MAX_ACCEL_C62X,
+	.qhw_num_engines = MAX_AE_C62X,
+	.qhw_tx_rx_gap = ETR_TX_RX_GAP_C62X,
+	.qhw_tx_rings_mask = ETR_TX_RINGS_MASK_C62X,
+	.qhw_clock_per_sec = CLOCK_PER_SEC_C62X,
+	.qhw_fw_auth = true,
+	.qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17,
+	.qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17,
+	.qhw_ring_asym_tx = 0,
+	.qhw_ring_asym_rx = 8,
+	.qhw_ring_sym_tx = 2,
+	.qhw_ring_sym_rx = 10,
+	.qhw_mof_fwname = AE_FW_MOF_NAME_C62X,
+	.qhw_mmp_fwname = AE_FW_MMP_NAME_C62X,
+	.qhw_prod_type = AE_FW_PROD_TYPE_C62X,
+	.qhw_get_accel_mask = qat_c62x_get_accel_mask,
+	.qhw_get_ae_mask = qat_c62x_get_ae_mask,
+	.qhw_get_sku = qat_c62x_get_sku,
+	.qhw_get_accel_cap = qat_c62x_get_accel_cap,
+	.qhw_get_fw_uof_name = qat_c62x_get_fw_uof_name,
+	.qhw_enable_intr = qat_c62x_enable_intr,
+	.qhw_init_admin_comms = qat_adm_mailbox_init,
+	.qhw_send_admin_init = qat_adm_mailbox_send_init,
+	.qhw_init_arb = qat_arb_init,
+	.qhw_get_arb_mapping = qat_c62x_get_arb_mapping,
+	.qhw_enable_error_correction = qat_c62x_enable_error_correction,
+	.qhw_disable_error_interrupts = qat_c62x_disable_error_interrupts,
+	.qhw_set_ssm_wdtimer = qat_set_ssm_wdtimer,
+	.qhw_check_slice_hang = qat_check_slice_hang,
+	.qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc,
+	.qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params,
+	.qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data),
+};
diff --git a/sys/dev/qat/qat_c62xreg.h b/sys/dev/qat/qat_c62xreg.h
new file mode 100644
index 000000000000..e052992fa6f5
--- /dev/null
+++ b/sys/dev/qat/qat_c62xreg.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_c62xreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_C62XREG_H_
+#define _DEV_PCI_QAT_C62XREG_H_
+
+/* Max number of accelerators and engines */
+#define MAX_ACCEL_C62X			5
+#define MAX_AE_C62X			10
+
+/* PCIe BAR index */
+#define BAR_SRAM_ID_C62X		0
+#define BAR_PMISC_ID_C62X		1
+#define BAR_ETR_ID_C62X			2
+
+/* BAR PMISC sub-regions */
+#define AE_OFFSET_C62X			0x20000
+#define AE_LOCAL_OFFSET_C62X		0x20800
+#define CAP_GLOBAL_OFFSET_C62X		0x30000
+
+#define SOFTSTRAP_REG_C62X			0x2EC
+#define SOFTSTRAP_SS_POWERGATE_CY_C62X		__BIT(23)
+#define SOFTSTRAP_SS_POWERGATE_PKE_C62X		__BIT(24)
+
+#define ACCEL_REG_OFFSET_C62X		16
+#define ACCEL_MASK_C62X			0x1F
+#define AE_MASK_C62X			0x3FF
+
+#define SMIAPF0_C62X			0x3A028
+#define SMIAPF1_C62X			0x3A030
+#define SMIA0_MASK_C62X			0xFFFF
+#define SMIA1_MASK_C62X			0x1
+
+/* Error detection and correction */
+#define AE_CTX_ENABLES_C62X(i)		((i) * 0x1000 + 0x20818)
+#define AE_MISC_CONTROL_C62X(i)		((i) * 0x1000 + 0x20960)
+#define ENABLE_AE_ECC_ERR_C62X		__BIT(28)
+#define ENABLE_AE_ECC_PARITY_CORR_C62X	(__BIT(24) | __BIT(12))
+#define ERRSSMSH_EN_C62X		__BIT(3)
+/* BIT(2) enables the logging of push/pull data errors. */
+#define PPERR_EN_C62X			(__BIT(2))
+
+/* Mask for VF2PF interrupts */
+#define VF2PF1_16_C62X			(0xFFFF << 9)
+#define ERRSOU3_VF2PF_C62X(errsou3)	(((errsou3) & 0x01FFFE00) >> 9)
+#define ERRMSK3_VF2PF_C62X(vf_mask)	(((vf_mask) & 0xFFFF) << 9)
+
+/* Masks for correctable error interrupts. */
+#define ERRMSK0_CERR_C62X	(__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0))
+#define ERRMSK1_CERR_C62X	(__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0))
+#define ERRMSK3_CERR_C62X	(__BIT(7))
+#define ERRMSK4_CERR_C62X	(__BIT(8) | __BIT(0))
+#define ERRMSK5_CERR_C62X	(0)
+
+/* Masks for uncorrectable error interrupts. */
+#define ERRMSK0_UERR_C62X	(__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1))
+#define ERRMSK1_UERR_C62X	(__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1))
+#define ERRMSK3_UERR_C62X	(__BIT(8) | __BIT(6) | __BIT(5) | __BIT(4) | \
+				 __BIT(3) | __BIT(2) | __BIT(0))
+#define ERRMSK4_UERR_C62X	(__BIT(9) | __BIT(1))
+#define ERRMSK5_UERR_C62X	(__BIT(18) | __BIT(17) | __BIT(16))
+
+/* RI CPP control */
+#define RICPPINTCTL_C62X		(0x3A000 + 0x110)
+/*
+ * BIT(2) enables error detection and reporting on the RI Parity Error.
+ * BIT(1) enables error detection and reporting on the RI CPP Pull interface.
+ * BIT(0) enables error detection and reporting on the RI CPP Push interface.
+ */
+#define RICPP_EN_C62X			(__BIT(2) | __BIT(1) | __BIT(0))
+
+/* TI CPP control */
+#define TICPPINTCTL_C62X		(0x3A400 + 0x138)
+/*
+ * BIT(3) enables error detection and reporting on the ETR Parity Error.
+ * BIT(2) enables error detection and reporting on the TI Parity Error.
+ * BIT(1) enables error detection and reporting on the TI CPP Pull interface.
+ * BIT(0) enables error detection and reporting on the TI CPP Push interface.
+ */
+#define TICPP_EN_C62X		\
+	(__BIT(4) | __BIT(3) | __BIT(2) | __BIT(1) | __BIT(0))
+
+/* CFC Uncorrectable Errors */
+#define CPP_CFC_ERR_CTRL_C62X	(0x30000 + 0xC00)
+/*
+ * BIT(1) enables interrupt.
+ * BIT(0) enables detecting and logging of push/pull data errors.
+ */
+#define CPP_CFC_UE_C62X		(__BIT(1) | __BIT(0))
+
+/* Correctable SecureRAM Error Reg */
+#define SECRAMCERR_C62X			(0x3AC00 + 0x00)
+/* BIT(3) enables fixing and logging of correctable errors. */
+#define SECRAM_CERR_C62X		(__BIT(3))
+
+/* Uncorrectable SecureRAM Error Reg */
+/*
+ * BIT(17) enables interrupt.
+ * BIT(3) enables detecting and logging of uncorrectable errors.
+ */
+#define SECRAM_UERR_C62X		(__BIT(17) | __BIT(3))
+
+/* Miscellaneous Memory Target Errors Register */
+/*
+ * BIT(3) enables detecting and logging push/pull data errors.
+ * BIT(2) enables interrupt.
+ */
+#define TGT_UERR_C62X			(__BIT(3) | __BIT(2))
+
+
+#define SLICEPWRDOWN_C62X(i)	((i) * 0x4000 + 0x2C)
+/* Enabling PKE4-PKE0. */
+#define MMP_PWR_UP_MSK_C62X		\
+	(__BIT(20) | __BIT(19) | __BIT(18) | __BIT(17) | __BIT(16))
+
+/* CPM Uncorrectable Errors */
+#define INTMASKSSM_C62X(i)		((i) * 0x4000 + 0x0)
+/* Disabling interrupts for correctable errors. */
+#define INTMASKSSM_UERR_C62X	\
+	(__BIT(11) | __BIT(9) | __BIT(7) | __BIT(5) | __BIT(3) | __BIT(1))
+
+/* MMP */
+/* BIT(3) enables correction. */
+#define CERRSSMMMP_EN_C62X		(__BIT(3))
+
+/* BIT(3) enables logging. */
+#define UERRSSMMMP_EN_C62X		(__BIT(3))
+
+/* ETR */
+#define ETR_MAX_BANKS_C62X		16
+#define ETR_TX_RX_GAP_C62X		8
+#define ETR_TX_RINGS_MASK_C62X		0xFF
+#define ETR_BUNDLE_SIZE_C62X		0x1000
+
+/* AE firmware */
+#define AE_FW_PROD_TYPE_C62X		0x01000000
+#define AE_FW_MOF_NAME_C62X	"qat_c62x"
+#define AE_FW_MMP_NAME_C62X	"qat_c62x_mmp"
+#define AE_FW_UOF_NAME_C62X	"icp_qat_ae.suof"
+
+/* Clock frequency */
+#define CLOCK_PER_SEC_C62X		(685 * 1000000 / 16)
+
+#endif
diff --git a/sys/dev/qat/qat_d15xx.c b/sys/dev/qat/qat_d15xx.c
new file mode 100644
index 000000000000..cc0effb75934
--- /dev/null
+++ b/sys/dev/qat/qat_d15xx.c
@@ -0,0 +1,314 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_d15xx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_d15xx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qat_hw17reg.h"
+#include "qat_d15xxreg.h"
+#include "qatvar.h"
+#include "qat_hw17var.h"
+
+static uint32_t
+qat_d15xx_get_accel_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, strap;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_D15XX, 4);
+
+	return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_D15XX) &
+	    ACCEL_MASK_D15XX);
+}
+
+static uint32_t
+qat_d15xx_get_ae_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, me_strap, me_disable, ssms_disabled;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	me_strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_D15XX, 4);
+
+	/* If SSMs are disabled, then disable the corresponding MEs */
+	ssms_disabled = (~qat_d15xx_get_accel_mask(sc)) & ACCEL_MASK_D15XX;
+	me_disable = 0x3;
+	while (ssms_disabled) {
+		if (ssms_disabled & 1)
+			me_strap |= me_disable;
+		ssms_disabled >>= 1;
+		me_disable <<= 2;
+	}
+
+	return (~(fusectl | me_strap)) & AE_MASK_D15XX;
+}
+
+static enum qat_sku
+qat_d15xx_get_sku(struct qat_softc *sc)
+{
+	switch (sc->sc_ae_num) {
+	case 8:
+		return QAT_SKU_2;
+	case MAX_AE_D15XX:
+		return QAT_SKU_4;
+	}
+
+	return QAT_SKU_UNKNOWN;
+}
+
+static uint32_t
+qat_d15xx_get_accel_cap(struct qat_softc *sc)
+{
+	uint32_t cap, legfuse, strap;
+
+	legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_D15XX, 4);
+
+	cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC +
+		QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC +
+		QAT_ACCEL_CAP_CIPHER +
+		QAT_ACCEL_CAP_AUTHENTICATION +
+		QAT_ACCEL_CAP_COMPRESSION +
+		QAT_ACCEL_CAP_ZUC +
+		QAT_ACCEL_CAP_SHA3;
+
+	if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) {
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC;
+		cap &= ~QAT_ACCEL_CAP_CIPHER;
+	}
+	if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE)
+		cap &= ~QAT_ACCEL_CAP_AUTHENTICATION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE)
+		cap &= ~QAT_ACCEL_CAP_ZUC;
+
+	if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_PKE_D15XX)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_CY_D15XX)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+
+	return cap;
+}
+
+static const char *
+qat_d15xx_get_fw_uof_name(struct qat_softc *sc)
+{
+
+	return AE_FW_UOF_NAME_D15XX;
+}
+
+static void
+qat_d15xx_enable_intr(struct qat_softc *sc)
+{
+
+	/* Enable bundle and misc interrupts */
+	qat_misc_write_4(sc, SMIAPF0_D15XX, SMIA0_MASK_D15XX);
+	qat_misc_write_4(sc, SMIAPF1_D15XX, SMIA1_MASK_D15XX);
+}
+
+/* Worker thread to service arbiter mappings */
+static uint32_t thrd_to_arb_map[] = {
+	0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA,
+	0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA
+};
+
+static void
+qat_d15xx_get_arb_mapping(struct qat_softc *sc, const uint32_t **arb_map_config)
+{
+	int i;
+
+	for (i = 1; i < MAX_AE_D15XX; i++) {
+		if ((~sc->sc_ae_mask) & (1 << i))
+			thrd_to_arb_map[i] = 0;
+	}
+	*arb_map_config = thrd_to_arb_map;
+}
+
+static void
+qat_d15xx_enable_error_interrupts(struct qat_softc *sc)
+{
+	qat_misc_write_4(sc, ERRMSK0, ERRMSK0_CERR_D15XX); /* ME0-ME3 */
+	qat_misc_write_4(sc, ERRMSK1, ERRMSK1_CERR_D15XX); /* ME4-ME7 */
+	qat_misc_write_4(sc, ERRMSK4, ERRMSK4_CERR_D15XX); /* ME8-ME9 */
+	qat_misc_write_4(sc, ERRMSK5, ERRMSK5_CERR_D15XX); /* SSM2-SSM4 */
+
+	/* Reset everything except VFtoPF1_16. */
+	qat_misc_read_write_and_4(sc, ERRMSK3, VF2PF1_16_D15XX);
+	/* Disable Secure RAM correctable error interrupt */
+	qat_misc_read_write_or_4(sc, ERRMSK3, ERRMSK3_CERR_D15XX);
+
+	/* RI CPP bus interface error detection and reporting. */
+	qat_misc_write_4(sc, RICPPINTCTL_D15XX, RICPP_EN_D15XX);
+
+	/* TI CPP bus interface error detection and reporting. */
+	qat_misc_write_4(sc, TICPPINTCTL_D15XX, TICPP_EN_D15XX);
+
+	/* Enable CFC Error interrupts and logging. */
+	qat_misc_write_4(sc, CPP_CFC_ERR_CTRL_D15XX, CPP_CFC_UE_D15XX);
+
+	/* Enable SecureRAM to fix and log Correctable errors */
+	qat_misc_write_4(sc, SECRAMCERR_D15XX, SECRAM_CERR_D15XX);
+
+	/* Enable SecureRAM Uncorrectable error interrupts and logging */
+	qat_misc_write_4(sc, SECRAMUERR, SECRAM_UERR_D15XX);
+
+	/* Enable Push/Pull Misc Uncorrectable error interrupts and logging */
+	qat_misc_write_4(sc, CPPMEMTGTERR, TGT_UERR_D15XX);
+}
+
+static void
+qat_d15xx_disable_error_interrupts(struct qat_softc *sc)
+{
+	/* ME0-ME3 */
+	qat_misc_write_4(sc, ERRMSK0, ERRMSK0_UERR_D15XX | ERRMSK0_CERR_D15XX);
+	/* ME4-ME7 */
+	qat_misc_write_4(sc, ERRMSK1, ERRMSK1_UERR_D15XX | ERRMSK1_CERR_D15XX);
+	/* Secure RAM, CPP Push Pull, RI, TI, SSM0-SSM1, CFC */
+	qat_misc_write_4(sc, ERRMSK3, ERRMSK3_UERR_D15XX | ERRMSK3_CERR_D15XX);
+	/* ME8-ME9 */
+	qat_misc_write_4(sc, ERRMSK4, ERRMSK4_UERR_D15XX | ERRMSK4_CERR_D15XX);
+	/* SSM2-SSM4 */
+	qat_misc_write_4(sc, ERRMSK5, ERRMSK5_UERR_D15XX | ERRMSK5_CERR_D15XX);
+}
+
+static void
+qat_d15xx_enable_error_correction(struct qat_softc *sc)
+{
+	u_int i, mask;
+
+	/* Enable Accel Engine error detection & correction */
+	for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_D15XX(i),
+		    ENABLE_AE_ECC_ERR_D15XX);
+		qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_D15XX(i),
+		    ENABLE_AE_ECC_PARITY_CORR_D15XX);
+	}
+
+	/* Enable shared memory error detection & correction */
+	for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_D15XX);
+		qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_D15XX);
+		qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_D15XX);
+	}
+
+	qat_d15xx_enable_error_interrupts(sc);
+}
+
+const struct qat_hw qat_hw_d15xx = {
+	.qhw_sram_bar_id = BAR_SRAM_ID_D15XX,
+	.qhw_misc_bar_id = BAR_PMISC_ID_D15XX,
+	.qhw_etr_bar_id = BAR_ETR_ID_D15XX,
+	.qhw_cap_global_offset = CAP_GLOBAL_OFFSET_D15XX,
+	.qhw_ae_offset = AE_OFFSET_D15XX,
+	.qhw_ae_local_offset = AE_LOCAL_OFFSET_D15XX,
+	.qhw_etr_bundle_size = ETR_BUNDLE_SIZE_D15XX,
+	.qhw_num_banks = ETR_MAX_BANKS_D15XX,
+	.qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK,
+	.qhw_num_accel = MAX_ACCEL_D15XX,
+	.qhw_num_engines = MAX_AE_D15XX,
+	.qhw_tx_rx_gap = ETR_TX_RX_GAP_D15XX,
+	.qhw_tx_rings_mask = ETR_TX_RINGS_MASK_D15XX,
+	.qhw_clock_per_sec = CLOCK_PER_SEC_D15XX,
+	.qhw_fw_auth = true,
+	.qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17,
+	.qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17,
+	.qhw_ring_asym_tx = 0,
+	.qhw_ring_asym_rx = 8,
+	.qhw_ring_sym_tx = 2,
+	.qhw_ring_sym_rx = 10,
+	.qhw_mof_fwname = AE_FW_MOF_NAME_D15XX,
+	.qhw_mmp_fwname = AE_FW_MMP_NAME_D15XX,
+	.qhw_prod_type = AE_FW_PROD_TYPE_D15XX,
+	.qhw_get_accel_mask = qat_d15xx_get_accel_mask,
+	.qhw_get_ae_mask = qat_d15xx_get_ae_mask,
+	.qhw_get_sku = qat_d15xx_get_sku,
+	.qhw_get_accel_cap = qat_d15xx_get_accel_cap,
+	.qhw_get_fw_uof_name = qat_d15xx_get_fw_uof_name,
+	.qhw_enable_intr = qat_d15xx_enable_intr,
+	.qhw_init_admin_comms = qat_adm_mailbox_init,
+	.qhw_send_admin_init = qat_adm_mailbox_send_init,
+	.qhw_init_arb = qat_arb_init,
+	.qhw_get_arb_mapping = qat_d15xx_get_arb_mapping,
+	.qhw_enable_error_correction = qat_d15xx_enable_error_correction,
+	.qhw_disable_error_interrupts = qat_d15xx_disable_error_interrupts,
+	.qhw_set_ssm_wdtimer = qat_set_ssm_wdtimer,
+	.qhw_check_slice_hang = qat_check_slice_hang,
+	.qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc,
+	.qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params,
+	.qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data),
+};
diff --git a/sys/dev/qat/qat_d15xxreg.h b/sys/dev/qat/qat_d15xxreg.h
new file mode 100644
index 000000000000..aefeb5fca04e
--- /dev/null
+++ b/sys/dev/qat/qat_d15xxreg.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_d15xxreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_D15XXREG_H_
+#define _DEV_PCI_QAT_D15XXREG_H_
+
+/* Max number of accelerators and engines */
+#define MAX_ACCEL_D15XX			5
+#define MAX_AE_D15XX			10
+
+/* PCIe BAR index */
+#define BAR_SRAM_ID_D15XX		0
+#define BAR_PMISC_ID_D15XX		1
+#define BAR_ETR_ID_D15XX		2
+
+/* BAR PMISC sub-regions */
+#define AE_OFFSET_D15XX			0x20000
+#define AE_LOCAL_OFFSET_D15XX		0x20800
+#define CAP_GLOBAL_OFFSET_D15XX		0x30000
+
+#define SOFTSTRAP_REG_D15XX			0x2EC
+#define SOFTSTRAP_SS_POWERGATE_CY_D15XX		__BIT(23)
+#define SOFTSTRAP_SS_POWERGATE_PKE_D15XX	__BIT(24)
+
+#define ACCEL_REG_OFFSET_D15XX		16
+#define ACCEL_MASK_D15XX		0x1F
+#define AE_MASK_D15XX			0x3FF
+
+#define SMIAPF0_D15XX			0x3A028
+#define SMIAPF1_D15XX			0x3A030
+#define SMIA0_MASK_D15XX		0xFFFF
+#define SMIA1_MASK_D15XX		0x1
+
+/* Error detection and correction */
+#define AE_CTX_ENABLES_D15XX(i)		((i) * 0x1000 + 0x20818)
+#define AE_MISC_CONTROL_D15XX(i)		((i) * 0x1000 + 0x20960)
+#define ENABLE_AE_ECC_ERR_D15XX		__BIT(28)
+#define ENABLE_AE_ECC_PARITY_CORR_D15XX	(__BIT(24) | __BIT(12))
+#define ERRSSMSH_EN_D15XX		__BIT(3)
+/* BIT(2) enables the logging of push/pull data errors. */
+#define PPERR_EN_D15XX			(__BIT(2))
+
+/* Mask for VF2PF interrupts */
+#define VF2PF1_16_D15XX			(0xFFFF << 9)
+#define ERRSOU3_VF2PF_D15XX(errsou3)	(((errsou3) & 0x01FFFE00) >> 9)
+#define ERRMSK3_VF2PF_D15XX(vf_mask)	(((vf_mask) & 0xFFFF) << 9)
+
+/* Masks for correctable error interrupts. */
+#define ERRMSK0_CERR_D15XX	(__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0))
+#define ERRMSK1_CERR_D15XX	(__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0))
+#define ERRMSK3_CERR_D15XX	(__BIT(7))
+#define ERRMSK4_CERR_D15XX	(__BIT(8) | __BIT(0))
+#define ERRMSK5_CERR_D15XX	(0)
+
+/* Masks for uncorrectable error interrupts. */
+#define ERRMSK0_UERR_D15XX	(__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1))
+#define ERRMSK1_UERR_D15XX	(__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1))
+#define ERRMSK3_UERR_D15XX	(__BIT(8) | __BIT(6) | __BIT(5) | __BIT(4) | \
+				 __BIT(3) | __BIT(2) | __BIT(0))
+#define ERRMSK4_UERR_D15XX	(__BIT(9) | __BIT(1))
+#define ERRMSK5_UERR_D15XX	(__BIT(18) | __BIT(17) | __BIT(16))
+
+/* RI CPP control */
+#define RICPPINTCTL_D15XX		(0x3A000 + 0x110)
+/*
+ * BIT(2) enables error detection and reporting on the RI Parity Error.
+ * BIT(1) enables error detection and reporting on the RI CPP Pull interface.
+ * BIT(0) enables error detection and reporting on the RI CPP Push interface.
+ */
+#define RICPP_EN_D15XX			(__BIT(2) | __BIT(1) | __BIT(0))
+
+/* TI CPP control */
+#define TICPPINTCTL_D15XX		(0x3A400 + 0x138)
+/*
+ * BIT(3) enables error detection and reporting on the ETR Parity Error.
+ * BIT(2) enables error detection and reporting on the TI Parity Error.
+ * BIT(1) enables error detection and reporting on the TI CPP Pull interface.
+ * BIT(0) enables error detection and reporting on the TI CPP Push interface.
+ */
+#define TICPP_EN_D15XX		\
+	(__BIT(4) | __BIT(3) | __BIT(2) | __BIT(1) | __BIT(0))
+
+/* CFC Uncorrectable Errors */
+#define CPP_CFC_ERR_CTRL_D15XX	(0x30000 + 0xC00)
+/*
+ * BIT(1) enables interrupt.
+ * BIT(0) enables detecting and logging of push/pull data errors.
+ */
+#define CPP_CFC_UE_D15XX		(__BIT(1) | __BIT(0))
+
+/* Correctable SecureRAM Error Reg */
+#define SECRAMCERR_D15XX			(0x3AC00 + 0x00)
+/* BIT(3) enables fixing and logging of correctable errors. */
+#define SECRAM_CERR_D15XX		(__BIT(3))
+
+/* Uncorrectable SecureRAM Error Reg */
+/*
+ * BIT(17) enables interrupt.
+ * BIT(3) enables detecting and logging of uncorrectable errors.
+ */
+#define SECRAM_UERR_D15XX		(__BIT(17) | __BIT(3))
+
+/* Miscellaneous Memory Target Errors Register */
+/*
+ * BIT(3) enables detecting and logging push/pull data errors.
+ * BIT(2) enables interrupt.
+ */
+#define TGT_UERR_D15XX			(__BIT(3) | __BIT(2))
+
+
+#define SLICEPWRDOWN_D15XX(i)	((i) * 0x4000 + 0x2C)
+/* Enabling PKE4-PKE0. */
+#define MMP_PWR_UP_MSK_D15XX		\
+	(__BIT(20) | __BIT(19) | __BIT(18) | __BIT(17) | __BIT(16))
+
+/* CPM Uncorrectable Errors */
+#define INTMASKSSM_D15XX(i)		((i) * 0x4000 + 0x0)
+/* Disabling interrupts for correctable errors. */
+#define INTMASKSSM_UERR_D15XX	\
+	(__BIT(11) | __BIT(9) | __BIT(7) | __BIT(5) | __BIT(3) | __BIT(1))
+
+/* MMP */
+/* BIT(3) enables correction. */
+#define CERRSSMMMP_EN_D15XX		(__BIT(3))
+
+/* BIT(3) enables logging. */
+#define UERRSSMMMP_EN_D15XX		(__BIT(3))
+
+/* ETR */
+#define ETR_MAX_BANKS_D15XX		16
+#define ETR_TX_RX_GAP_D15XX		8
+#define ETR_TX_RINGS_MASK_D15XX		0xFF
+#define ETR_BUNDLE_SIZE_D15XX		0x1000
+
+/* AE firmware */
+#define AE_FW_PROD_TYPE_D15XX	0x01000000
+#define AE_FW_MOF_NAME_D15XX	"qat_d15xx"
+#define AE_FW_MMP_NAME_D15XX	"qat_d15xx_mmp"
+#define AE_FW_UOF_NAME_D15XX	"icp_qat_ae.suof"
+
+/* Clock frequency */
+#define CLOCK_PER_SEC_D15XX		(685 * 1000000 / 16)
+
+#endif
diff --git a/sys/dev/qat/qat_dh895xcc.c b/sys/dev/qat/qat_dh895xcc.c
new file mode 100644
index 000000000000..8b566f3925d4
--- /dev/null
+++ b/sys/dev/qat/qat_dh895xcc.c
@@ -0,0 +1,271 @@
+/* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
+/*
+ * Copyright (c) 2020 Rubicon Communications, LLC (Netgate)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 - 2020 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+
+#include <machine/bus.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qatvar.h"
+#include "qat_hw17reg.h"
+#include "qat_hw17var.h"
+#include "qat_dh895xccreg.h"
+
+static uint32_t
+qat_dh895xcc_get_accel_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, strap;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_DH895XCC, 4);
+
+	return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_DH895XCC) &
+	    ACCEL_MASK_DH895XCC);
+}
+
+static uint32_t
+qat_dh895xcc_get_ae_mask(struct qat_softc *sc)
+{
+	uint32_t fusectl, strap;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_DH895XCC, 4);
+
+	return (~(fusectl | strap)) & AE_MASK_DH895XCC;
+}
+
+static enum qat_sku
+qat_dh895xcc_get_sku(struct qat_softc *sc)
+{
+	uint32_t fusectl, sku;
+
+	fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4);
+	sku = (fusectl & FUSECTL_SKU_MASK_DH895XCC) >>
+	    FUSECTL_SKU_SHIFT_DH895XCC;
+	switch (sku) {
+	case FUSECTL_SKU_1_DH895XCC:
+		return QAT_SKU_1;
+	case FUSECTL_SKU_2_DH895XCC:
+		return QAT_SKU_2;
+	case FUSECTL_SKU_3_DH895XCC:
+		return QAT_SKU_3;
+	case FUSECTL_SKU_4_DH895XCC:
+		return QAT_SKU_4;
+	default:
+		return QAT_SKU_UNKNOWN;
+	}
+}
+
+static uint32_t
+qat_dh895xcc_get_accel_cap(struct qat_softc *sc)
+{
+	uint32_t cap, legfuse;
+
+	legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4);
+
+	cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC +
+		QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC +
+		QAT_ACCEL_CAP_CIPHER +
+		QAT_ACCEL_CAP_AUTHENTICATION +
+		QAT_ACCEL_CAP_COMPRESSION +
+		QAT_ACCEL_CAP_ZUC +
+		QAT_ACCEL_CAP_SHA3;
+
+	if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) {
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC;
+		cap &= ~QAT_ACCEL_CAP_CIPHER;
+	}
+	if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE)
+		cap &= ~QAT_ACCEL_CAP_AUTHENTICATION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE)
+		cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC;
+	if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE)
+		cap &= ~QAT_ACCEL_CAP_COMPRESSION;
+	if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE)
+		cap &= ~QAT_ACCEL_CAP_ZUC;
+
+	return cap;
+}
+
+static const char *
+qat_dh895xcc_get_fw_uof_name(struct qat_softc *sc)
+{
+	return AE_FW_UOF_NAME_DH895XCC;
+}
+
+static void
+qat_dh895xcc_enable_intr(struct qat_softc *sc)
+{
+	/* Enable bundle and misc interrupts */
+	qat_misc_write_4(sc, SMIAPF0_DH895XCC, SMIA0_MASK_DH895XCC);
+	qat_misc_write_4(sc, SMIAPF1_DH895XCC, SMIA1_MASK_DH895XCC);
+}
+
+/* Worker thread to service arbiter mappings based on dev SKUs */
+static uint32_t thrd_to_arb_map_sku4[] = {
+	0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666,
+	0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000,
+};
+
+static uint32_t thrd_to_arb_map_sku6[] = {
+	0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666,
+	0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222,
+	0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222,
+};
+
+static void
+qat_dh895xcc_get_arb_mapping(struct qat_softc *sc,
+    const uint32_t **arb_map_config)
+{
+	uint32_t *map, sku;
+	int i;
+
+	sku = qat_dh895xcc_get_sku(sc);
+	switch (sku) {
+	case QAT_SKU_1:
+		map = thrd_to_arb_map_sku4;
+		break;
+	case QAT_SKU_2:
+	case QAT_SKU_4:
+		map = thrd_to_arb_map_sku6;
+		break;
+	default:
+		*arb_map_config = NULL;
+		return;
+	}
+
+	for (i = 1; i < MAX_AE_DH895XCC; i++) {
+		if ((~sc->sc_ae_mask) & (1 << i))
+			map[i] = 0;
+	}
+	*arb_map_config = map;
+}
+
+static void
+qat_dh895xcc_enable_error_correction(struct qat_softc *sc)
+{
+	uint32_t mask;
+	u_int i;
+
+	/* Enable Accel Engine error detection & correction */
+	for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_DH895XCC(i),
+		    ENABLE_AE_ECC_ERR_DH895XCC);
+		qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_DH895XCC(i),
+		    ENABLE_AE_ECC_PARITY_CORR_DH895XCC);
+	}
+
+	/* Enable shared memory error detection & correction */
+	for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_DH895XCC);
+		qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_DH895XCC);
+		qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_DH895XCC);
+	}
+}
+
+const struct qat_hw qat_hw_dh895xcc = {
+	.qhw_sram_bar_id = BAR_SRAM_ID_DH895XCC,
+	.qhw_misc_bar_id = BAR_PMISC_ID_DH895XCC,
+	.qhw_etr_bar_id = BAR_ETR_ID_DH895XCC,
+	.qhw_cap_global_offset = CAP_GLOBAL_OFFSET_DH895XCC,
+	.qhw_ae_offset = AE_OFFSET_DH895XCC,
+	.qhw_ae_local_offset = AE_LOCAL_OFFSET_DH895XCC,
+	.qhw_etr_bundle_size = ETR_BUNDLE_SIZE_DH895XCC,
+	.qhw_num_banks = ETR_MAX_BANKS_DH895XCC,
+	.qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK,
+	.qhw_num_accel = MAX_ACCEL_DH895XCC,
+	.qhw_num_engines = MAX_AE_DH895XCC,
+	.qhw_tx_rx_gap = ETR_TX_RX_GAP_DH895XCC,
+	.qhw_tx_rings_mask = ETR_TX_RINGS_MASK_DH895XCC,
+	.qhw_clock_per_sec = CLOCK_PER_SEC_DH895XCC,
+	.qhw_fw_auth = false,
+	.qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17,
+	.qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17,
+	.qhw_ring_asym_tx = 0,
+	.qhw_ring_asym_rx = 8,
+	.qhw_ring_sym_tx = 2,
+	.qhw_ring_sym_rx = 10,
+	.qhw_mof_fwname = AE_FW_MOF_NAME_DH895XCC,
+	.qhw_mmp_fwname = AE_FW_MMP_NAME_DH895XCC,
+	.qhw_prod_type = AE_FW_PROD_TYPE_DH895XCC,
+	.qhw_get_accel_mask = qat_dh895xcc_get_accel_mask,
+	.qhw_get_ae_mask = qat_dh895xcc_get_ae_mask,
+	.qhw_get_sku = qat_dh895xcc_get_sku,
+	.qhw_get_accel_cap = qat_dh895xcc_get_accel_cap,
+	.qhw_get_fw_uof_name = qat_dh895xcc_get_fw_uof_name,
+	.qhw_enable_intr = qat_dh895xcc_enable_intr,
+	.qhw_init_admin_comms = qat_adm_mailbox_init,
+	.qhw_send_admin_init = qat_adm_mailbox_send_init,
+	.qhw_init_arb = qat_arb_init,
+	.qhw_get_arb_mapping = qat_dh895xcc_get_arb_mapping,
+	.qhw_enable_error_correction = qat_dh895xcc_enable_error_correction,
+	.qhw_check_slice_hang = qat_check_slice_hang,
+	.qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc,
+	.qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params,
+	.qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data),
+};
diff --git a/sys/dev/qat/qat_dh895xccreg.h b/sys/dev/qat/qat_dh895xccreg.h
new file mode 100644
index 000000000000..2657f7ea4211
--- /dev/null
+++ b/sys/dev/qat/qat_dh895xccreg.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014-2020 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_DH895XCCREG_H_
+#define _DEV_PCI_QAT_DH895XCCREG_H_
+
+/* Max number of accelerators and engines */
+#define MAX_ACCEL_DH895XCC		6
+#define MAX_AE_DH895XCC			12
+
+/* PCIe BAR index */
+#define BAR_SRAM_ID_DH895XCC		0
+#define BAR_PMISC_ID_DH895XCC		1
+#define BAR_ETR_ID_DH895XCC		2
+
+/* BAR PMISC sub-regions */
+#define AE_OFFSET_DH895XCC		0x20000
+#define AE_LOCAL_OFFSET_DH895XCC	0x20800
+#define CAP_GLOBAL_OFFSET_DH895XCC	0x30000
+
+#define SOFTSTRAP_REG_DH895XCC			0x2EC
+
+#define	FUSECTL_SKU_MASK_DH895XCC	0x300000
+#define	FUSECTL_SKU_SHIFT_DH895XCC	20
+#define	FUSECTL_SKU_1_DH895XCC		0
+#define	FUSECTL_SKU_2_DH895XCC		1
+#define	FUSECTL_SKU_3_DH895XCC		2
+#define	FUSECTL_SKU_4_DH895XCC		3
+
+#define ACCEL_REG_OFFSET_DH895XCC	13
+#define ACCEL_MASK_DH895XCC		0x3F
+#define AE_MASK_DH895XCC		0xFFF
+
+#define SMIAPF0_DH895XCC		0x3A028
+#define SMIAPF1_DH895XCC		0x3A030
+#define SMIA0_MASK_DH895XCC		0xFFFFFFFF
+#define SMIA1_MASK_DH895XCC		0x1
+
+/* Error detection and correction */
+#define AE_CTX_ENABLES_DH895XCC(i)	((i) * 0x1000 + 0x20818)
+#define AE_MISC_CONTROL_DH895XCC(i)	((i) * 0x1000 + 0x20960)
+#define ENABLE_AE_ECC_ERR_DH895XCC	__BIT(28)
+#define ENABLE_AE_ECC_PARITY_CORR_DH895XCC (__BIT(24) | __BIT(12))
+#define ERRSSMSH_EN_DH895XCC		__BIT(3)
+/* BIT(2) enables the logging of push/pull data errors. */
+#define PPERR_EN_DH895XCC		(__BIT(2))
+
+/* ETR */
+#define ETR_MAX_BANKS_DH895XCC		32
+#define ETR_TX_RX_GAP_DH895XCC		8
+#define ETR_TX_RINGS_MASK_DH895XCC	0xFF
+#define ETR_BUNDLE_SIZE_DH895XCC	0x1000
+
+/* AE firmware */
+#define AE_FW_PROD_TYPE_DH895XCC	0x00400000
+#define AE_FW_MOF_NAME_DH895XCC		"qat_895xcc"
+#define AE_FW_MMP_NAME_DH895XCC		"qat_895xcc_mmp"
+#define AE_FW_UOF_NAME_DH895XCC		"icp_qat_ae.uof"
+
+/* Clock frequency */
+#define CLOCK_PER_SEC_DH895XCC		(685 * 1000000 / 16)
+
+#endif
diff --git a/sys/dev/qat/qat_hw15.c b/sys/dev/qat/qat_hw15.c
new file mode 100644
index 000000000000..4f823a0a9ae4
--- /dev/null
+++ b/sys/dev/qat/qat_hw15.c
@@ -0,0 +1,953 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2013 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+
+#include <opencrypto/xform.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qat_hw15reg.h"
+#include "qatvar.h"
+#include "qat_hw15var.h"
+
+static int	qat_adm_ring_init_ring_table(struct qat_softc *);
+static void	qat_adm_ring_build_slice_mask(uint16_t *, uint32_t, uint32_t);
+static void	qat_adm_ring_build_shram_mask(uint64_t *, uint32_t, uint32_t);
+static int	qat_adm_ring_build_ring_table(struct qat_softc *, uint32_t);
+static int	qat_adm_ring_build_init_msg(struct qat_softc *,
+		    struct fw_init_req *, enum fw_init_cmd_id, uint32_t,
+		    struct qat_accel_init_cb *);
+static int	qat_adm_ring_send_init_msg_sync(struct qat_softc *,
+		    enum fw_init_cmd_id, uint32_t);
+static int	qat_adm_ring_send_init_msg(struct qat_softc *,
+		    enum fw_init_cmd_id);
+static int	qat_adm_ring_intr(struct qat_softc *, void *, void *);
+
+void
+qat_msg_req_type_populate(struct arch_if_req_hdr *msg, enum arch_if_req type,
+    uint32_t rxring)
+{
+
+	memset(msg, 0, sizeof(struct arch_if_req_hdr));
+	msg->flags = ARCH_IF_FLAGS_VALID_FLAG |
+	    ARCH_IF_FLAGS_RESP_RING_TYPE_ET | ARCH_IF_FLAGS_RESP_TYPE_S;
+	msg->req_type = type;
+	msg->resp_pipe_id = rxring;
+}
+
+void
+qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *msg, bus_addr_t desc_paddr,
+    uint8_t hdrsz, uint8_t hwblksz, uint16_t comn_req_flags, uint32_t flow_id)
+{
+	struct fw_comn_req_hdr *hdr = &msg->comn_hdr;
+
+	hdr->comn_req_flags = comn_req_flags;
+	hdr->content_desc_params_sz = hwblksz;
+	hdr->content_desc_hdr_sz = hdrsz;
+	hdr->content_desc_addr = desc_paddr;
+	msg->flow_id = flow_id;
+}
+
+void
+qat_msg_service_cmd_populate(struct fw_la_bulk_req *msg, enum fw_la_cmd_id cmdid,
+    uint16_t cmd_flags)
+{
+	msg->comn_la_req.la_cmd_id = cmdid;
+	msg->comn_la_req.u.la_flags = cmd_flags;
+}
+
+void
+qat_msg_cmn_mid_populate(struct fw_comn_req_mid *msg, void *cookie,
+    uint64_t src, uint64_t dst)
+{
+
+	msg->opaque_data = (uint64_t)(uintptr_t)cookie;
+	msg->src_data_addr = src;
+	if (dst == 0)
+		msg->dest_data_addr = src;
+	else
+		msg->dest_data_addr = dst;
+}
+
+void
+qat_msg_req_params_populate(struct fw_la_bulk_req *msg,
+    bus_addr_t req_params_paddr, uint8_t req_params_sz)
+{
+	msg->req_params_addr = req_params_paddr;
+	msg->comn_la_req.u1.req_params_blk_sz = req_params_sz / 8;
+}
+
+void
+qat_msg_cmn_footer_populate(union fw_comn_req_ftr *msg, uint64_t next_addr)
+{
+	msg->next_request_addr = next_addr;
+}
+
+void
+qat_msg_params_populate(struct fw_la_bulk_req *msg,
+    struct qat_crypto_desc *desc, uint8_t req_params_sz,
+    uint16_t service_cmd_flags, uint16_t comn_req_flags)
+{
+	qat_msg_cmn_hdr_populate(msg, desc->qcd_desc_paddr,
+	    desc->qcd_hdr_sz, desc->qcd_hw_blk_sz, comn_req_flags, 0);
+	qat_msg_service_cmd_populate(msg, desc->qcd_cmd_id, service_cmd_flags);
+	qat_msg_cmn_mid_populate(&msg->comn_mid, NULL, 0, 0);
+	qat_msg_req_params_populate(msg, 0, req_params_sz);
+	qat_msg_cmn_footer_populate(&msg->comn_ftr, 0);
+}
+
+static int
+qat_adm_ring_init_ring_table(struct qat_softc *sc)
+{
+	struct qat_admin_rings *qadr = &sc->sc_admin_rings;
+
+	if (sc->sc_ae_num == 1) {
+		qadr->qadr_cya_ring_tbl =
+		    &qadr->qadr_master_ring_tbl[0];
+		qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A;
+	} else if (sc->sc_ae_num == 2 || sc->sc_ae_num == 4) {
+		qadr->qadr_cya_ring_tbl =
+		    &qadr->qadr_master_ring_tbl[0];
+		qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A;
+		qadr->qadr_cyb_ring_tbl =
+		    &qadr->qadr_master_ring_tbl[1];
+		qadr->qadr_srv_mask[1] = QAT_SERVICE_CRYPTO_B;
+	}
+
+	return 0;
+}
+
+int
+qat_adm_ring_init(struct qat_softc *sc)
+{
+	struct qat_admin_rings *qadr = &sc->sc_admin_rings;
+	int error, i, j;
+
+	error = qat_alloc_dmamem(sc, &qadr->qadr_dma, 1, PAGE_SIZE, PAGE_SIZE);
+	if (error)
+		return error;
+
+	qadr->qadr_master_ring_tbl = qadr->qadr_dma.qdm_dma_vaddr;
+
+	MPASS(sc->sc_ae_num *
+	    sizeof(struct fw_init_ring_table) <= PAGE_SIZE);
+
+	/* Initialize the Master Ring Table */
+	for (i = 0; i < sc->sc_ae_num; i++) {
+		struct fw_init_ring_table *firt =
+		    &qadr->qadr_master_ring_tbl[i];
+
+		for (j = 0; j < INIT_RING_TABLE_SZ; j++) {
+			struct fw_init_ring_params *firp = 
+			    &firt->firt_bulk_rings[j];
+
+			firp->firp_reserved = 0;
+			firp->firp_curr_weight = QAT_DEFAULT_RING_WEIGHT;
+			firp->firp_init_weight = QAT_DEFAULT_RING_WEIGHT;
+			firp->firp_ring_pvl = QAT_DEFAULT_PVL;
+		}
+		memset(firt->firt_ring_mask, 0, sizeof(firt->firt_ring_mask));
+	}
+
+	error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_TX,
+	    ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_req_size,
+	    NULL, NULL, "admin_tx", &qadr->qadr_admin_tx);
+	if (error)
+		return error;
+
+	error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_RX,
+	    ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_resp_size,
+	    qat_adm_ring_intr, qadr, "admin_rx", &qadr->qadr_admin_rx);
+	if (error)
+		return error;
+
+	/*
+	 * Finally set up the service indices into the Master Ring Table
+	 * and convenient ring table pointers for each service enabled.
+	 * Only the Admin rings are initialized.
+	 */
+	error = qat_adm_ring_init_ring_table(sc);
+	if (error)
+		return error;
+
+	/*
+	 * Calculate the number of active AEs per QAT
+	 * needed for Shram partitioning.
+	 */
+	for (i = 0; i < sc->sc_ae_num; i++) {
+		if (qadr->qadr_srv_mask[i])
+			qadr->qadr_active_aes_per_accel++;
+	}
+
+	return 0;
+}
+
+static void
+qat_adm_ring_build_slice_mask(uint16_t *slice_mask, uint32_t srv_mask,
+   uint32_t init_shram)
+{
+	uint16_t shram = 0, comn_req = 0;
+
+	if (init_shram)
+		shram = COMN_REQ_SHRAM_INIT_REQUIRED;
+
+	if (srv_mask & QAT_SERVICE_CRYPTO_A)
+		comn_req |= COMN_REQ_CY0_ONLY(shram);
+	if (srv_mask & QAT_SERVICE_CRYPTO_B)
+		comn_req |= COMN_REQ_CY1_ONLY(shram);
+
+	*slice_mask = comn_req;
+}
+
+static void
+qat_adm_ring_build_shram_mask(uint64_t *shram_mask, uint32_t active_aes,
+    uint32_t ae)
+{
+	*shram_mask = 0;
+
+	if (active_aes == 1) {
+		*shram_mask = ~(*shram_mask);
+	} else if (active_aes == 2) {
+		if (ae == 1)
+			*shram_mask = ((~(*shram_mask)) & 0xffffffff);
+		else
+			*shram_mask = ((~(*shram_mask)) & 0xffffffff00000000ull);
+	} else if (active_aes == 3) {
+		if (ae == 0)
+			*shram_mask = ((~(*shram_mask)) & 0x7fffff);
+		else if (ae == 1)
+			*shram_mask = ((~(*shram_mask)) & 0x3fffff800000ull);
+		else
+			*shram_mask = ((~(*shram_mask)) & 0xffffc00000000000ull);
+	} else {
+		panic("Only three services are supported in current version");
+	}
+}
+
+static int
+qat_adm_ring_build_ring_table(struct qat_softc *sc, uint32_t ae)
+{
+	struct qat_admin_rings *qadr = &sc->sc_admin_rings;
+	struct fw_init_ring_table *tbl;
+	struct fw_init_ring_params *param;
+	uint8_t srv_mask = sc->sc_admin_rings.qadr_srv_mask[ae];
+
+	if ((srv_mask & QAT_SERVICE_CRYPTO_A)) {
+		tbl = qadr->qadr_cya_ring_tbl;
+	} else if ((srv_mask & QAT_SERVICE_CRYPTO_B)) {
+		tbl = qadr->qadr_cyb_ring_tbl;
+	} else {
+		device_printf(sc->sc_dev,
+		    "Invalid execution engine %d\n", ae);
+		return EINVAL;
+	}
+
+	param = &tbl->firt_bulk_rings[sc->sc_hw.qhw_ring_sym_tx];
+	param->firp_curr_weight = QAT_HI_PRIO_RING_WEIGHT;
+	param->firp_init_weight = QAT_HI_PRIO_RING_WEIGHT;
+	FW_INIT_RING_MASK_SET(tbl, sc->sc_hw.qhw_ring_sym_tx);
+
+	return 0;
+}
+
+static int
+qat_adm_ring_build_init_msg(struct qat_softc *sc,
+    struct fw_init_req *initmsg, enum fw_init_cmd_id cmd, uint32_t ae,
+    struct qat_accel_init_cb *cb)
+{
+	struct fw_init_set_ae_info_hdr *aehdr;
+	struct fw_init_set_ae_info *aeinfo;
+	struct fw_init_set_ring_info_hdr *ringhdr;
+	struct fw_init_set_ring_info *ringinfo;
+	int init_shram = 0, tgt_id, cluster_id;
+	uint32_t srv_mask;
+
+	srv_mask = sc->sc_admin_rings.qadr_srv_mask[
+	    ae % sc->sc_ae_num];
+	
+	memset(initmsg, 0, sizeof(struct fw_init_req));
+
+	qat_msg_req_type_populate(&initmsg->comn_hdr.arch_if,
+	    ARCH_IF_REQ_QAT_FW_INIT,
+	    sc->sc_admin_rings.qadr_admin_rx->qr_ring_id);
+
+	qat_msg_cmn_mid_populate(&initmsg->comn_mid, cb, 0, 0);
+
+	switch (cmd) {
+	case FW_INIT_CMD_SET_AE_INFO:
+		if (ae % sc->sc_ae_num == 0)
+			init_shram = 1;
+		if (ae >= sc->sc_ae_num) {
+			tgt_id = 1;
+			cluster_id = 1;
+		} else {
+			cluster_id = 0;
+			if (sc->sc_ae_mask)
+				tgt_id = 0;
+			else
+				tgt_id = 1;
+		}
+		aehdr = &initmsg->u.set_ae_info;
+		aeinfo = &initmsg->u1.set_ae_info;
+
+		aehdr->init_cmd_id = cmd;
+		/* XXX that does not support sparse ae_mask */
+		aehdr->init_trgt_id = ae;
+		aehdr->init_ring_cluster_id = cluster_id;
+		aehdr->init_qat_id = tgt_id;
+
+		qat_adm_ring_build_slice_mask(&aehdr->init_slice_mask, srv_mask,
+		    init_shram);
+
+		qat_adm_ring_build_shram_mask(&aeinfo->init_shram_mask,
+		    sc->sc_admin_rings.qadr_active_aes_per_accel,
+		    ae % sc->sc_ae_num);
+
+		break;
+	case FW_INIT_CMD_SET_RING_INFO:
+		ringhdr = &initmsg->u.set_ring_info;
+		ringinfo = &initmsg->u1.set_ring_info;
+
+		ringhdr->init_cmd_id = cmd;
+		/* XXX that does not support sparse ae_mask */
+		ringhdr->init_trgt_id = ae;
+
+		/* XXX */
+		qat_adm_ring_build_ring_table(sc,
+		    ae % sc->sc_ae_num);
+
+		ringhdr->init_ring_tbl_sz = sizeof(struct fw_init_ring_table);
+
+		ringinfo->init_ring_table_ptr =
+		    sc->sc_admin_rings.qadr_dma.qdm_dma_seg.ds_addr +
+		    ((ae % sc->sc_ae_num) *
+		    sizeof(struct fw_init_ring_table));
+
+		break;
+	default:
+		return ENOTSUP;
+	}
+
+	return 0;
+}
+
+static int
+qat_adm_ring_send_init_msg_sync(struct qat_softc *sc,
+    enum fw_init_cmd_id cmd, uint32_t ae)
+{
+	struct fw_init_req initmsg;
+	struct qat_accel_init_cb cb;
+	int error;
+
+	error = qat_adm_ring_build_init_msg(sc, &initmsg, cmd, ae, &cb);
+	if (error)
+		return error;
+
+	error = qat_etr_put_msg(sc, sc->sc_admin_rings.qadr_admin_tx,
+	    (uint32_t *)&initmsg);
+	if (error)
+		return error;
+
+	error = tsleep(&cb, PZERO, "qat_init", hz * 3 / 2);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "Timed out initialization firmware: %d\n", error);
+		return error;
+	}
+	if (cb.qaic_status) {
+		device_printf(sc->sc_dev, "Failed to initialize firmware\n");
+		return EIO;
+	}
+
+	return error;
+}
+
+static int
+qat_adm_ring_send_init_msg(struct qat_softc *sc,
+    enum fw_init_cmd_id cmd)
+{
+	struct qat_admin_rings *qadr = &sc->sc_admin_rings;
+	uint32_t error, ae;
+
+	for (ae = 0; ae < sc->sc_ae_num; ae++) {
+		uint8_t srv_mask = qadr->qadr_srv_mask[ae];
+		switch (cmd) {
+		case FW_INIT_CMD_SET_AE_INFO:
+		case FW_INIT_CMD_SET_RING_INFO:
+			if (!srv_mask)
+				continue;
+			break;
+		case FW_INIT_CMD_TRNG_ENABLE:
+		case FW_INIT_CMD_TRNG_DISABLE:
+			if (!(srv_mask & QAT_SERVICE_CRYPTO_A))
+				continue;
+			break;
+		default:
+			return ENOTSUP;
+		}
+
+		error = qat_adm_ring_send_init_msg_sync(sc, cmd, ae);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
+int
+qat_adm_ring_send_init(struct qat_softc *sc)
+{
+	int error;
+
+	error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_AE_INFO);
+	if (error)
+		return error;
+
+	error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_RING_INFO);
+	if (error)
+		return error;
+
+	return 0;
+}
+
+static int
+qat_adm_ring_intr(struct qat_softc *sc, void *arg, void *msg)
+{
+	struct arch_if_resp_hdr *resp;
+	struct fw_init_resp *init_resp;
+	struct qat_accel_init_cb *init_cb;
+	int handled = 0;
+
+	resp = (struct arch_if_resp_hdr *)msg;
+
+	switch (resp->resp_type) {
+	case ARCH_IF_REQ_QAT_FW_INIT:
+		init_resp = (struct fw_init_resp *)msg;
+		init_cb = (struct qat_accel_init_cb *)
+		    (uintptr_t)init_resp->comn_resp.opaque_data;
+		init_cb->qaic_status =
+		    __SHIFTOUT(init_resp->comn_resp.comn_status,
+		    COMN_RESP_INIT_ADMIN_STATUS);
+		wakeup(init_cb);
+		break;
+	default:
+		device_printf(sc->sc_dev,
+		    "unknown resp type %d\n", resp->resp_type);
+		break;
+	}
+
+	return handled;
+}
+
+static inline uint16_t
+qat_hw15_get_comn_req_flags(uint8_t ae)
+{
+	if (ae == 0) {
+		return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL |
+		    COMN_REQ_AUTH0_SLICE_REQUIRED |
+		    COMN_REQ_CIPHER0_SLICE_REQUIRED;
+	} else {
+		return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL |
+		    COMN_REQ_AUTH1_SLICE_REQUIRED |
+		    COMN_REQ_CIPHER1_SLICE_REQUIRED;
+	}
+}
+
+static uint32_t
+qat_hw15_crypto_setup_cipher_desc(struct qat_crypto_desc *desc,
+    struct qat_session *qs, struct fw_cipher_hdr *cipher_hdr,
+    uint32_t hw_blk_offset, enum fw_slice next_slice)
+{
+	desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ;
+
+	cipher_hdr->state_padding_sz = 0;
+	cipher_hdr->key_sz = qs->qs_cipher_klen / 8;
+
+	cipher_hdr->state_sz = desc->qcd_cipher_blk_sz / 8;
+
+	cipher_hdr->next_id = next_slice;
+	cipher_hdr->curr_id = FW_SLICE_CIPHER;
+	cipher_hdr->offset = hw_blk_offset / 8;
+	cipher_hdr->resrvd = 0;
+
+	return sizeof(struct hw_cipher_config) + qs->qs_cipher_klen;
+}
+
+static void
+qat_hw15_crypto_setup_cipher_config(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, const struct cryptop *crp,
+    struct hw_cipher_config *cipher_config)
+{
+	const uint8_t *key;
+	uint8_t *cipher_key;
+
+	cipher_config->val = qat_crypto_load_cipher_session(desc, qs);
+	cipher_config->reserved = 0;
+
+	cipher_key = (uint8_t *)(cipher_config + 1);
+	if (crp != NULL && crp->crp_cipher_key != NULL)
+		key = crp->crp_cipher_key;
+	else
+		key = qs->qs_cipher_key;
+	memcpy(cipher_key, key, qs->qs_cipher_klen);
+}
+
+static uint32_t
+qat_hw15_crypto_setup_auth_desc(struct qat_crypto_desc *desc,
+    struct qat_session *qs, struct fw_auth_hdr *auth_hdr,
+    uint32_t ctrl_blk_offset, uint32_t hw_blk_offset,
+    enum fw_slice next_slice)
+{
+	const struct qat_sym_hash_def *hash_def;
+
+	(void)qat_crypto_load_auth_session(desc, qs, &hash_def);
+
+	auth_hdr->next_id = next_slice;
+	auth_hdr->curr_id = FW_SLICE_AUTH;
+	auth_hdr->offset = hw_blk_offset / 8;
+	auth_hdr->resrvd = 0;
+
+	auth_hdr->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED;
+	auth_hdr->u.inner_prefix_sz = 0;
+	auth_hdr->outer_prefix_sz = 0;
+	auth_hdr->final_sz = hash_def->qshd_alg->qshai_digest_len;
+	auth_hdr->inner_state1_sz =
+	    roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
+	auth_hdr->inner_res_sz = hash_def->qshd_alg->qshai_digest_len;
+	auth_hdr->inner_state2_sz =
+	    roundup(hash_def->qshd_qat->qshqi_state2_len, 8);
+	auth_hdr->inner_state2_off = auth_hdr->offset +
+	    ((sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz) / 8);
+
+	auth_hdr->outer_config_off = 0;
+	auth_hdr->outer_state1_sz = 0;
+	auth_hdr->outer_res_sz = 0;
+	auth_hdr->outer_prefix_off = 0;
+
+	desc->qcd_auth_sz = hash_def->qshd_alg->qshai_sah->hashsize;
+	desc->qcd_state_storage_sz = (sizeof(struct hw_auth_counter) +
+	    roundup(hash_def->qshd_alg->qshai_state_size, 8)) / 8;
+	desc->qcd_gcm_aad_sz_offset1 = desc->qcd_auth_offset +
+	    sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz +
+	    AES_BLOCK_LEN;
+	desc->qcd_gcm_aad_sz_offset2 = ctrl_blk_offset +
+	    offsetof(struct fw_auth_hdr, u.aad_sz);
+
+	return sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz +
+	    auth_hdr->inner_state2_sz;
+}
+
+static void
+qat_hw15_crypto_setup_auth_setup(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, const struct cryptop *crp,
+    struct hw_auth_setup *auth_setup)
+{
+	const struct qat_sym_hash_def *hash_def;
+	const uint8_t *key;
+	uint8_t *state1, *state2;
+	uint32_t state_sz, state1_sz, state2_sz, state1_pad_len, state2_pad_len;
+
+	auth_setup->auth_config.config = qat_crypto_load_auth_session(desc, qs,
+	    &hash_def);
+	auth_setup->auth_config.reserved = 0;
+
+	auth_setup->auth_counter.counter =
+	    htobe32(hash_def->qshd_qat->qshqi_auth_counter);
+	auth_setup->auth_counter.reserved = 0;
+
+	state1 = (uint8_t *)(auth_setup + 1);
+	state2 = state1 + roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
+	switch (qs->qs_auth_algo) {
+	case HW_AUTH_ALGO_GALOIS_128:
+		qat_crypto_gmac_precompute(desc, qs->qs_cipher_key,
+		    qs->qs_cipher_klen, hash_def, state2);
+		break;
+	case HW_AUTH_ALGO_SHA1:
+		state_sz = hash_def->qshd_alg->qshai_state_size;
+		state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
+		state2_sz = roundup(hash_def->qshd_qat->qshqi_state2_len, 8);
+		if (qs->qs_auth_mode == HW_AUTH_MODE1) {
+			state1_pad_len = state1_sz - state_sz;
+			state2_pad_len = state2_sz - state_sz;
+			if (state1_pad_len > 0)
+				memset(state1 + state_sz, 0, state1_pad_len);
+			if (state2_pad_len > 0)
+				memset(state2 + state_sz, 0, state2_pad_len);
+		}
+		/* FALLTHROUGH */
+	case HW_AUTH_ALGO_SHA256:
+	case HW_AUTH_ALGO_SHA384:
+	case HW_AUTH_ALGO_SHA512:
+		switch (qs->qs_auth_mode) {
+		case HW_AUTH_MODE0:
+			memcpy(state1, hash_def->qshd_alg->qshai_init_state,
+			    state1_sz);
+			/* Override for mode 0 hashes. */
+			auth_setup->auth_counter.counter = 0;
+			break;
+		case HW_AUTH_MODE1:
+			if (crp != NULL && crp->crp_auth_key != NULL)
+				key = crp->crp_auth_key;
+			else
+				key = qs->qs_auth_key;
+			if (key != NULL) {
+				qat_crypto_hmac_precompute(desc, key,
+				    qs->qs_auth_klen, hash_def, state1, state2);
+			}
+			break;
+		default:
+			panic("%s: unhandled auth mode %d", __func__,
+			    qs->qs_auth_mode);
+		}
+		break;
+	default:
+		panic("%s: unhandled auth algorithm %d", __func__,
+		    qs->qs_auth_algo);
+	}
+}
+
+void
+qat_hw15_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs,
+    struct qat_crypto_desc *desc)
+{
+	struct fw_cipher_hdr *cipher_hdr;
+	struct fw_auth_hdr *auth_hdr;
+	struct fw_la_bulk_req *req_cache;
+	struct hw_auth_setup *auth_setup;
+	struct hw_cipher_config *cipher_config;
+	uint32_t ctrl_blk_sz, ctrl_blk_offset, hw_blk_offset;
+	int i;
+	uint16_t la_cmd_flags;
+	uint8_t req_params_sz;
+	uint8_t *ctrl_blk_ptr, *hw_blk_ptr;
+
+	ctrl_blk_sz = 0;
+	if (qs->qs_cipher_algo != HW_CIPHER_ALGO_NULL)
+		ctrl_blk_sz += sizeof(struct fw_cipher_hdr);
+	if (qs->qs_auth_algo != HW_AUTH_ALGO_NULL)
+		ctrl_blk_sz += sizeof(struct fw_auth_hdr);
+
+	ctrl_blk_ptr = desc->qcd_content_desc;
+	ctrl_blk_offset = 0;
+	hw_blk_ptr = ctrl_blk_ptr + ctrl_blk_sz;
+	hw_blk_offset = 0;
+
+	la_cmd_flags = 0;
+	req_params_sz = 0;
+	for (i = 0; i < MAX_FW_SLICE; i++) {
+		switch (desc->qcd_slices[i]) {
+		case FW_SLICE_CIPHER:
+			cipher_hdr = (struct fw_cipher_hdr *)(ctrl_blk_ptr +
+			    ctrl_blk_offset);
+			cipher_config = (struct hw_cipher_config *)(hw_blk_ptr +
+			    hw_blk_offset);
+			desc->qcd_cipher_offset = ctrl_blk_sz + hw_blk_offset;
+			hw_blk_offset += qat_hw15_crypto_setup_cipher_desc(desc,
+			    qs, cipher_hdr, hw_blk_offset,
+			    desc->qcd_slices[i + 1]);
+			qat_hw15_crypto_setup_cipher_config(desc, qs, NULL,
+			    cipher_config);
+			ctrl_blk_offset += sizeof(struct fw_cipher_hdr);
+			req_params_sz += sizeof(struct fw_la_cipher_req_params);
+			break;
+		case FW_SLICE_AUTH:
+			auth_hdr = (struct fw_auth_hdr *)(ctrl_blk_ptr +
+			    ctrl_blk_offset);
+			auth_setup = (struct hw_auth_setup *)(hw_blk_ptr +
+			    hw_blk_offset);
+			desc->qcd_auth_offset = ctrl_blk_sz + hw_blk_offset;
+			hw_blk_offset += qat_hw15_crypto_setup_auth_desc(desc,
+			    qs, auth_hdr, ctrl_blk_offset, hw_blk_offset,
+			    desc->qcd_slices[i + 1]);
+			qat_hw15_crypto_setup_auth_setup(desc, qs, NULL,
+			    auth_setup);
+			ctrl_blk_offset += sizeof(struct fw_auth_hdr);
+			req_params_sz += sizeof(struct fw_la_auth_req_params);
+			la_cmd_flags |= LA_FLAGS_RET_AUTH_RES;
+			/* no digest verify */
+			break;
+		case FW_SLICE_DRAM_WR:
+			i = MAX_FW_SLICE; /* end of chain */
+			break;
+		default:
+			MPASS(0);
+			break;
+		}
+	}
+
+	desc->qcd_hdr_sz = ctrl_blk_offset / 8;
+	desc->qcd_hw_blk_sz = hw_blk_offset / 8;
+
+	req_cache = (struct fw_la_bulk_req *)desc->qcd_req_cache;
+	qat_msg_req_type_populate(
+	    &req_cache->comn_hdr.arch_if,
+	    ARCH_IF_REQ_QAT_FW_LA, 0);
+
+	if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128)
+		la_cmd_flags |= LA_FLAGS_PROTO_GCM | LA_FLAGS_GCM_IV_LEN_FLAG;
+	else
+		la_cmd_flags |= LA_FLAGS_PROTO_NO;
+
+	qat_msg_params_populate(req_cache, desc, req_params_sz,
+	    la_cmd_flags, 0);
+
+	bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
+	    qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE);
+}
+
+static void
+qat_hw15_crypto_req_setkey(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, struct qat_sym_cookie *qsc,
+    struct fw_la_bulk_req *bulk_req, struct cryptop *crp)
+{
+	struct hw_auth_setup *auth_setup;
+	struct hw_cipher_config *cipher_config;
+	uint8_t *cdesc;
+	int i;
+
+	cdesc = qsc->qsc_content_desc;
+	memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE);
+	for (i = 0; i < MAX_FW_SLICE; i++) {
+		switch (desc->qcd_slices[i]) {
+		case FW_SLICE_CIPHER:
+			cipher_config = (struct hw_cipher_config *)
+			    (cdesc + desc->qcd_cipher_offset);
+			qat_hw15_crypto_setup_cipher_config(desc, qs, crp,
+			    cipher_config);
+			break;
+		case FW_SLICE_AUTH:
+			auth_setup = (struct hw_auth_setup *)
+			    (cdesc + desc->qcd_auth_offset);
+			qat_hw15_crypto_setup_auth_setup(desc, qs, crp,
+			    auth_setup);
+			break;
+		case FW_SLICE_DRAM_WR:
+			i = MAX_FW_SLICE; /* end of chain */
+			break;
+		default:
+			MPASS(0);
+		}
+	}
+
+	bulk_req->comn_hdr.content_desc_addr = qsc->qsc_content_desc_paddr;
+}
+
+void
+qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *qcb,
+    struct qat_session *qs, struct qat_crypto_desc const *desc,
+    struct qat_sym_cookie *qsc, struct cryptop *crp)
+{
+	struct qat_sym_bulk_cookie *qsbc;
+	struct fw_la_bulk_req *bulk_req;
+	struct fw_la_cipher_req_params *cipher_req;
+	struct fw_la_auth_req_params *auth_req;
+	bus_addr_t digest_paddr;
+	uint8_t *aad_szp2, *req_params_ptr;
+	uint32_t aad_sz, *aad_szp1;
+	enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id;
+	enum fw_slice next_slice;
+
+	qsbc = &qsc->u.qsc_bulk_cookie;
+
+	bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg;
+	memcpy(bulk_req, &desc->qcd_req_cache, QAT_HW15_SESSION_REQ_CACHE_SIZE);
+	bulk_req->comn_hdr.arch_if.resp_pipe_id = qcb->qcb_sym_rx->qr_ring_id;
+	bulk_req->comn_hdr.comn_req_flags =
+	    qat_hw15_get_comn_req_flags(qcb->qcb_bank % 2);
+	bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr;
+	bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr;
+	bulk_req->req_params_addr = qsc->qsc_bulk_req_params_buf_paddr;
+	bulk_req->comn_ftr.next_request_addr = 0;
+	bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc;
+	if (__predict_false(crp->crp_cipher_key != NULL ||
+	    crp->crp_auth_key != NULL)) {
+		qat_hw15_crypto_req_setkey(desc, qs, qsc, bulk_req, crp);
+	}
+
+	digest_paddr = 0;
+	if (desc->qcd_auth_sz != 0)
+		digest_paddr = qsc->qsc_auth_res_paddr;
+
+	req_params_ptr = qsbc->qsbc_req_params_buf;
+	memset(req_params_ptr, 0, sizeof(qsbc->qsbc_req_params_buf));
+
+	/*
+	 * The SG list layout is a bit different for GCM and GMAC, it's simpler
+	 * to handle those cases separately.
+	 */
+	if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
+		cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr;
+		auth_req = (struct fw_la_auth_req_params *)
+		    (req_params_ptr + sizeof(struct fw_la_cipher_req_params));
+
+		cipher_req->cipher_state_sz = desc->qcd_cipher_blk_sz / 8;
+		cipher_req->curr_id = FW_SLICE_CIPHER;
+		if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH)
+			cipher_req->next_id = FW_SLICE_DRAM_WR;
+		else
+			cipher_req->next_id = FW_SLICE_AUTH;
+		cipher_req->state_address = qsc->qsc_iv_buf_paddr;
+
+		if (cmd_id != FW_LA_CMD_AUTH) {
+			/*
+			 * Don't fill out the cipher block if we're doing GMAC
+			 * only.
+			 */
+			cipher_req->cipher_off = 0;
+			cipher_req->cipher_len = crp->crp_payload_length;
+		}
+		auth_req->curr_id = FW_SLICE_AUTH;
+		if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH)
+			auth_req->next_id = FW_SLICE_CIPHER;
+		else
+			auth_req->next_id = FW_SLICE_DRAM_WR;
+
+		auth_req->auth_res_address = digest_paddr;
+		auth_req->auth_res_sz = desc->qcd_auth_sz;
+
+		auth_req->auth_off = 0;
+		auth_req->auth_len = crp->crp_payload_length;
+
+		auth_req->hash_state_sz =
+		    roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) >> 3;
+		auth_req->u1.aad_addr = crp->crp_aad_length > 0 ?
+		    qsc->qsc_gcm_aad_paddr : 0;
+
+		/*
+		 * Update the hash state block if necessary.  This only occurs
+		 * when the AAD length changes between requests in a session and
+		 * is synchronized by qat_process().
+		 */
+		aad_sz = htobe32(crp->crp_aad_length);
+		aad_szp1 = (uint32_t *)(
+		    __DECONST(uint8_t *, desc->qcd_content_desc) +
+		    desc->qcd_gcm_aad_sz_offset1);
+		aad_szp2 = __DECONST(uint8_t *, desc->qcd_content_desc) +
+		    desc->qcd_gcm_aad_sz_offset2;
+		if (__predict_false(*aad_szp1 != aad_sz)) {
+			*aad_szp1 = aad_sz;
+			*aad_szp2 = (uint8_t)roundup2(crp->crp_aad_length,
+			    QAT_AES_GCM_AAD_ALIGN);
+			bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
+			    qs->qs_desc_mem.qdm_dma_map,
+			    BUS_DMASYNC_PREWRITE);
+		}
+	} else {
+		cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr;
+		if (cmd_id != FW_LA_CMD_AUTH) {
+			if (cmd_id == FW_LA_CMD_CIPHER ||
+			    cmd_id == FW_LA_CMD_HASH_CIPHER)
+				next_slice = FW_SLICE_DRAM_WR;
+			else
+				next_slice = FW_SLICE_AUTH;
+
+			cipher_req->cipher_state_sz =
+			    desc->qcd_cipher_blk_sz / 8;
+
+			cipher_req->curr_id = FW_SLICE_CIPHER;
+			cipher_req->next_id = next_slice;
+
+			cipher_req->cipher_off = crp->crp_aad_length == 0 ? 0 :
+			    crp->crp_payload_start - crp->crp_aad_start;
+			cipher_req->cipher_len = crp->crp_payload_length;
+			cipher_req->state_address = qsc->qsc_iv_buf_paddr;
+		}
+		if (cmd_id != FW_LA_CMD_CIPHER) {
+			if (cmd_id == FW_LA_CMD_AUTH)
+				auth_req = (struct fw_la_auth_req_params *)
+				    req_params_ptr;
+			else
+				auth_req = (struct fw_la_auth_req_params *)
+				    (cipher_req + 1);
+			if (cmd_id == FW_LA_CMD_HASH_CIPHER)
+				next_slice = FW_SLICE_CIPHER;
+			else
+				next_slice = FW_SLICE_DRAM_WR;
+
+			auth_req->curr_id = FW_SLICE_AUTH;
+			auth_req->next_id = next_slice;
+
+			auth_req->auth_res_address = digest_paddr;
+			auth_req->auth_res_sz = desc->qcd_auth_sz;
+
+			auth_req->auth_len =
+			    crp->crp_payload_length + crp->crp_aad_length;
+			auth_req->auth_off = 0;
+
+			auth_req->hash_state_sz = 0;
+			auth_req->u1.prefix_addr = desc->qcd_hash_state_paddr +
+			    desc->qcd_state_storage_sz;
+		}
+	}
+}
diff --git a/sys/dev/qat/qat_hw15reg.h b/sys/dev/qat/qat_hw15reg.h
new file mode 100644
index 000000000000..df3fa14a0d85
--- /dev/null
+++ b/sys/dev/qat/qat_hw15reg.h
@@ -0,0 +1,635 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_hw15reg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2013 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_HW15REG_H_
+#define _DEV_PCI_QAT_HW15REG_H_
+
+/* Default message size in bytes */
+#define FW_REQ_DEFAULT_SZ_HW15		64
+#define FW_RESP_DEFAULT_SZ_HW15		64
+
+#define ADMIN_RING_SIZE		256
+#define RING_NUM_ADMIN_TX	0
+#define RING_NUM_ADMIN_RX	1
+
+/* -------------------------------------------------------------------------- */
+/* accel */
+
+#define ARCH_IF_FLAGS_VALID_FLAG		__BIT(7)
+#define ARCH_IF_FLAGS_RESP_RING_TYPE		__BITS(4, 3)
+#define  ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT	3
+#define  ARCH_IF_FLAGS_RESP_RING_TYPE_SCRATCH	(0 << ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT)
+#define  ARCH_IF_FLAGS_RESP_RING_TYPE_NN	(1 << ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT)
+#define  ARCH_IF_FLAGS_RESP_RING_TYPE_ET	(2 << ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT)
+#define ARCH_IF_FLAGS_RESP_TYPE			__BITS(2, 0)
+#define  ARCH_IF_FLAGS_RESP_TYPE_SHIFT		0
+#define   ARCH_IF_FLAGS_RESP_TYPE_A		(0 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT)
+#define   ARCH_IF_FLAGS_RESP_TYPE_B		(1 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT)
+#define   ARCH_IF_FLAGS_RESP_TYPE_C		(2 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT)
+#define   ARCH_IF_FLAGS_RESP_TYPE_S		(3 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT)
+
+enum arch_if_req {
+	ARCH_IF_REQ_NULL,		/* NULL request type */
+
+	/* QAT-AE Service Request Type IDs - 01 to 20 */
+	ARCH_IF_REQ_QAT_FW_INIT,	/* QAT-FW Initialization Request */
+	ARCH_IF_REQ_QAT_FW_ADMIN,	/* QAT-FW Administration Request */
+	ARCH_IF_REQ_QAT_FW_PKE,		/* QAT-FW PKE Request */
+	ARCH_IF_REQ_QAT_FW_LA,		/* QAT-FW Lookaside Request */
+	ARCH_IF_REQ_QAT_FW_IPSEC,	/* QAT-FW IPSec Request */
+	ARCH_IF_REQ_QAT_FW_SSL,		/* QAT-FW SSL Request */
+	ARCH_IF_REQ_QAT_FW_DMA,		/* QAT-FW DMA Request */
+	ARCH_IF_REQ_QAT_FW_STORAGE,	/* QAT-FW Storage Request */
+	ARCH_IF_REQ_QAT_FW_COMPRESS,	/* QAT-FW Compression Request */
+	ARCH_IF_REQ_QAT_FW_PATMATCH,	/* QAT-FW Pattern Matching Request */
+
+	/* IP Service (Range Match and Exception) Blocks Request Type IDs 21 - 30 */
+	ARCH_IF_REQ_RM_FLOW_MISS = 21,	/* RM flow miss request */
+	ARCH_IF_REQ_RM_FLOW_TIMER_EXP,	/* RM flow timer exp Request */
+	ARCH_IF_REQ_IP_SERVICES_RFC_LOOKUP_UPDATE, /* RFC Lookup request */
+	ARCH_IF_REQ_IP_SERVICES_CONFIG_UPDATE,	/* Config Update request */
+	ARCH_IF_REQ_IP_SERVICES_FCT_CONFIG,	/* FCT Config request */
+	ARCH_IF_REQ_IP_SERVICES_NEXT_HOP_TIMER_EXPIRY, /* NH Timer expiry request */
+	ARCH_IF_REQ_IP_SERVICES_EXCEPTION,	/* Exception processign request */
+	ARCH_IF_REQ_IP_SERVICES_STACK_DRIVER,	/* Send to SD request */
+	ARCH_IF_REQ_IP_SERVICES_ACTION_HANDLER,	/* Send to AH request */
+	ARCH_IF_REQ_IP_SERVICES_EVENT_HANDLER,	/* Send to EH request */
+	ARCH_IF_REQ_DELIMITER			/* End delimiter */
+};
+
+struct arch_if_req_hdr {
+	uint8_t resp_dest_id;
+	/* Opaque identifier passed from the request to response to allow
+	 * response handler perform any further processing */
+	uint8_t resp_pipe_id;
+	/* Response pipe to write the response associated with this request to */
+	uint8_t req_type;
+	/* Definition of the service described by the request */
+	uint8_t flags;
+	/* Request and response control flags */
+};
+
+struct arch_if_resp_hdr {
+	uint8_t dest_id;
+	/* Opaque identifier passed from the request to response to allow
+	 * response handler perform any further processing */
+	uint8_t serv_id;
+	/* Definition of the service id generating the response */
+	uint8_t resp_type;
+	/* Definition of the service described by the request */
+	uint8_t flags;
+	/* Request and response control flags */
+};
+
+struct fw_comn_req_hdr {
+	struct arch_if_req_hdr arch_if;
+	/* Common arch fields used by all ICP interface requests. Remaining
+	 * fields are specific to the common QAT FW service. */
+	uint16_t comn_req_flags;
+	/* Flags used to describe common processing required by the request and
+	 * the meaning of parameters in it i.e. differentiating between a buffer
+	 * descriptor and a flat buffer pointer in the source (src) and destination
+	 * (dest) data address fields. Full definition of the fields is given
+	 * below */
+	uint8_t content_desc_params_sz;
+	/* Size of the content descriptor parameters in quad words. These
+	 * parameters describe the session setup configuration info for the
+	 * slices that this request relies upon i.e. the configuration word and
+	 * cipher key needed by the cipher slice if there is a request for cipher
+	 * processing. The format of the parameters are contained in icp_qat_hw.h
+	 * and vary depending on the algorithm and mode being used. It is the
+	 * clients responsibility to ensure this structure is correctly packed */
+	uint8_t content_desc_hdr_sz;
+	/* Size of the content descriptor header in quad words. This information
+	 * is read into the QAT AE xfr registers */
+	uint64_t content_desc_addr;
+	/* Address of the content descriptor containing both the content header
+	 * the size of which is defined by content_desc_hdr_sz followed by the
+	 * content parameters whose size is described bycontent_desc_params_sz
+	 */
+};
+
+struct fw_comn_req_mid {
+	uint64_t opaque_data;
+	/* Opaque data passed unmodified from the request to response messages
+	 * by firmware (fw) */
+	uint64_t src_data_addr;
+	/* Generic definition of the source data supplied to the QAT AE. The
+	 * common flags are used to further describe the attributes of this
+	 * field */
+	uint64_t dest_data_addr;
+	/* Generic definition of the destination data supplied to the QAT AE.
+	 * The common flags are used to further describe the attributes of this
+	 * field */
+};
+
+union fw_comn_req_ftr {
+	uint64_t next_request_addr;
+	/* Overloaded field, for stateful requests, this field is the pointer to 
+	   next request descriptor */
+	struct {
+		uint32_t src_length;
+		/* Length of source flat buffer incase src buffer type is flat */
+		uint32_t dst_length;
+		/* Length of source flat buffer incase dst buffer type is flat */
+	} s;
+};
+
+union fw_comn_error {
+	struct {
+		uint8_t resrvd;		/* 8 bit reserved field */
+		uint8_t comn_err_code;	/**< 8 bit common error code */
+	} s;
+	/* Structure which is used for non-compression responses */
+
+	struct {
+		uint8_t xlat_err_code;	/* 8 bit translator error field */
+		uint8_t cmp_err_code;	/* 8 bit compression error field */
+	} s1;
+	/* Structure which is used for compression responses */
+};
+
+struct fw_comn_resp_hdr {
+	struct arch_if_resp_hdr arch_if;
+	/* Common arch fields used by all ICP interface response messages. The
+	 * remaining fields are specific to the QAT FW */
+	union fw_comn_error comn_error;
+	/* This field is overloaded to allow for one 8 bit common error field
+	 * or two 8 bit error fields from compression and translator  */
+	uint8_t comn_status;
+	/* Status field which specifies which slice(s) report an error */
+	uint8_t serv_cmd_id;
+	/* For services that define multiple commands this field represents the
+	 * command. If only 1 command is supported then this field will be 0 */
+	uint64_t opaque_data;
+	/* Opaque data passed from the request to the response message */
+};
+
+
+#define RING_MASK_TABLE_ENTRY_LOG_SZ			(5)
+
+#define FW_INIT_RING_MASK_SET(table, id)			\
+		table->firt_ring_mask[id >> RING_MASK_TABLE_ENTRY_LOG_SZ] =\
+		table->firt_ring_mask[id >> RING_MASK_TABLE_ENTRY_LOG_SZ] | \
+		(1 << (id & 0x1f))
+
+struct fw_init_ring_params {
+	uint8_t firp_curr_weight;	/* Current ring weight (working copy),
+					 * has to be equal to init_weight */
+	uint8_t firp_init_weight;	/* Initial ring weight: -1 ... 0
+					 * -1 is equal to FF, -2 is equal to FE,
+					 * the weighting uses negative logic
+					 * where FF means poll the ring once,
+					 * -2 is poll the ring twice,
+					 * 0 is poll the ring 255 times */
+	uint8_t firp_ring_pvl;		/* Ring Privilege Level. */
+	uint8_t firp_reserved;		/* Reserved field which must be set
+					 * to 0 by the client */
+};
+
+#define INIT_RING_TABLE_SZ		128
+#define INIT_RING_TABLE_LW_SZ		4
+
+struct fw_init_ring_table {
+	struct fw_init_ring_params firt_bulk_rings[INIT_RING_TABLE_SZ];
+					/* array of ring parameters */
+	uint32_t firt_ring_mask[INIT_RING_TABLE_LW_SZ];
+					/* Structure to hold the bit masks for
+					 * 128 rings. */
+};
+
+struct fw_init_set_ae_info_hdr {
+	uint16_t init_slice_mask;	/* Init time flags to set the ownership of the slices */
+	uint16_t resrvd;		/* Reserved field and must be set to 0 by the client */
+	uint8_t init_qat_id;		/* Init time qat id described in the request */
+	uint8_t init_ring_cluster_id;	/* Init time ring cluster Id */
+	uint8_t init_trgt_id;		/* Init time target AE id described in the request */
+	uint8_t init_cmd_id;		/* Init time command that is described in the request */
+};
+
+struct fw_init_set_ae_info {
+	uint64_t init_shram_mask;	/* Init time shram mask to set the page ownership in page pool of AE*/
+	uint64_t resrvd;		/* Reserved field and must be set to 0 by the client */
+};
+
+struct fw_init_set_ring_info_hdr {
+	uint32_t resrvd;		/* Reserved field and must be set to 0 by the client */
+	uint16_t init_ring_tbl_sz;	/* Init time information to state size of the ring table */
+	uint8_t init_trgt_id;		/* Init time target AE id described in the request */
+	uint8_t init_cmd_id;		/* Init time command that is described in the request */
+};
+
+struct fw_init_set_ring_info {
+	uint64_t init_ring_table_ptr;	/* Pointer to weighting information for 128 rings  */
+	uint64_t resrvd;		/* Reserved field and must be set to 0 by the client */
+};
+
+struct fw_init_trng_hdr {
+	uint32_t resrvd;	/* Reserved field and must be set to 0 by the client */
+	union {
+		uint8_t resrvd;	/* Reserved field set to if cmd type is trng disable */
+		uint8_t init_trng_cfg_sz;	/* Size of the trng config word in QW*/
+	} u;
+	uint8_t resrvd1;	/* Reserved field and must be set to 0 by the client */
+	uint8_t init_trgt_id;	/* Init time target AE id described in the request */
+	uint8_t init_cmd_id;	/* Init time command that is described in the request */
+};
+
+struct fw_init_trng {
+	union {
+		uint64_t resrvd;	/* Reserved field set to 0 if cmd type is trng disable */
+		uint64_t init_trng_cfg_ptr;	/* Pointer to TRNG Slice config word*/
+	} u;
+	uint64_t resrvd;		/* Reserved field and must be set to 0 by the client */
+};
+
+struct fw_init_req {
+	struct fw_comn_req_hdr comn_hdr;	/* Common request header */
+	union {
+		struct fw_init_set_ae_info_hdr set_ae_info;
+		/* INIT SET_AE_INFO request header structure */
+		struct fw_init_set_ring_info_hdr set_ring_info;
+		/* INIT SET_RING_INFO request header structure */
+		struct fw_init_trng_hdr init_trng;
+		/* INIT TRNG ENABLE/DISABLE request header structure */
+	} u;
+	struct fw_comn_req_mid comn_mid;	/* Common request middle section */
+	union {
+		struct fw_init_set_ae_info set_ae_info;
+		/* INIT SET_AE_INFO request data structure */
+		struct fw_init_set_ring_info set_ring_info;
+		/* INIT SET_RING_INFO request data structure */
+		struct fw_init_trng init_trng;
+		/* INIT TRNG ENABLE/DISABLE request data structure */
+	} u1;
+};
+
+enum fw_init_cmd_id {
+	FW_INIT_CMD_SET_AE_INFO,	/* Setup AE Info command type */
+	FW_INIT_CMD_SET_RING_INFO,	/* Setup Ring Info command type */
+	FW_INIT_CMD_TRNG_ENABLE,	/* TRNG Enable command type */
+	FW_INIT_CMD_TRNG_DISABLE,	/* TRNG Disable command type */
+	FW_INIT_CMD_DELIMITER		/* Delimiter type */
+};
+
+struct fw_init_resp {
+	struct fw_comn_resp_hdr comn_resp;	/* Common interface response */
+	uint8_t resrvd[64 - sizeof(struct fw_comn_resp_hdr)];
+	/* XXX FW_RESP_DEFAULT_SZ_HW15 */
+	/* Reserved padding out to the default response size */
+};
+
+/* -------------------------------------------------------------------------- */
+/* look aside */
+
+#define COMN_REQ_ORD				UINT16_C(0x8000)
+#define  COMN_REQ_ORD_SHIFT			15
+#define   COMN_REQ_ORD_NONE			(0 << COMN_REQ_ORD_SHIFT)
+#define   COMN_REQ_ORD_STRICT			(1 << COMN_REQ_ORD_SHIFT)
+#define COMN_REQ_PTR_TYPE			UINT16_C(0x4000)
+#define  COMN_REQ_PTR_TYPE_SHIFT		14
+#define   COMN_REQ_PTR_TYPE_FLAT		(0 << COMN_REQ_PTR_TYPE_SHIFT)
+#define   COMN_REQ_PTR_TYPE_SGL			(1 << COMN_REQ_PTR_TYPE_SHIFT)
+#define COMN_REQ_RESERVED			UINT16_C(0x2000)
+#define COMN_REQ_SHRAM_INIT			UINT16_C(0x1000)
+#define  COMN_REQ_SHRAM_INIT_SHIFT		12
+#define  COMN_REQ_SHRAM_INIT_REQUIRED		(1 << COMN_REQ_SHRAM_INIT_SHIFT)
+#define COMN_REQ_REGEX_SLICE			UINT16_C(0x0800)
+#define  COMN_REQ_REGEX_SLICE_SHIFT		11
+#define   COMN_REQ_REGEX_SLICE_REQUIRED		(1 << COMN_REQ_REGEX_SLICE_SHIFT)
+#define COMN_REQ_XLAT_SLICE			UINT16_C(0x0400)
+#define  COMN_REQ_XLAT_SLICE_SHIFT		10
+#define   COMN_REQ_XLAT_SLICE_REQUIRED		(1 << COMN_REQ_XLAT_SLICE_SHIFT)
+#define COMN_REQ_CPR_SLICE			UINT16_C(0x0200)
+#define  COMN_REQ_CPR_SLICE_SHIFT		9
+#define   COMN_REQ_CPR_SLICE_REQUIRED		(1 << COMN_REQ_CPR_SLICE_SHIFT)
+#define COMN_REQ_BULK_SLICE			UINT16_C(0x0100)
+#define  COMN_REQ_BULK_SLICE_SHIFT		8
+#define   COMN_REQ_BULK_SLICE_REQUIRED		(1 << COMN_REQ_BULK_SLICE_SHIFT)
+#define COMN_REQ_STORAGE_SLICE			UINT16_C(0x0080)
+#define  COMN_REQ_STORAGE_SLICE_SHIFT		7
+#define   COMN_REQ_STORAGE_SLICE_REQUIRED	(1 << COMN_REQ_STORAGE_SLICE_SHIFT)
+#define COMN_REQ_RND_SLICE			UINT16_C(0x0040)
+#define  COMN_REQ_RND_SLICE_SHIFT		6
+#define   COMN_REQ_RND_SLICE_REQUIRED		(1 << COMN_REQ_RND_SLICE_SHIFT)
+#define COMN_REQ_PKE1_SLICE			UINT16_C(0x0020)
+#define  COMN_REQ_PKE1_SLICE_SHIFT		5
+#define   COMN_REQ_PKE1_SLICE_REQUIRED		(1 << COMN_REQ_PKE1_SLICE_SHIFT)
+#define COMN_REQ_PKE0_SLICE			UINT16_C(0x0010)
+#define  COMN_REQ_PKE0_SLICE_SHIFT		4
+#define   COMN_REQ_PKE0_SLICE_REQUIRED		(1 << COMN_REQ_PKE0_SLICE_SHIFT)
+#define COMN_REQ_AUTH1_SLICE			UINT16_C(0x0008)
+#define  COMN_REQ_AUTH1_SLICE_SHIFT		3
+#define   COMN_REQ_AUTH1_SLICE_REQUIRED		(1 << COMN_REQ_AUTH1_SLICE_SHIFT)
+#define COMN_REQ_AUTH0_SLICE			UINT16_C(0x0004)
+#define  COMN_REQ_AUTH0_SLICE_SHIFT		2
+#define   COMN_REQ_AUTH0_SLICE_REQUIRED		(1 << COMN_REQ_AUTH0_SLICE_SHIFT)
+#define COMN_REQ_CIPHER1_SLICE			UINT16_C(0x0002)
+#define  COMN_REQ_CIPHER1_SLICE_SHIFT		1
+#define   COMN_REQ_CIPHER1_SLICE_REQUIRED	(1 << COMN_REQ_CIPHER1_SLICE_SHIFT)
+#define COMN_REQ_CIPHER0_SLICE			UINT16_C(0x0001)
+#define  COMN_REQ_CIPHER0_SLICE_SHIFT		0
+#define   COMN_REQ_CIPHER0_SLICE_REQUIRED	(1 << COMN_REQ_CIPHER0_SLICE_SHIFT)
+
+#define COMN_REQ_CY0_ONLY(shram)			\
+		COMN_REQ_ORD_STRICT |			\
+		COMN_REQ_PTR_TYPE_FLAT |		\
+		(shram) |				\
+		COMN_REQ_RND_SLICE_REQUIRED | 		\
+		COMN_REQ_PKE0_SLICE_REQUIRED |		\
+		COMN_REQ_AUTH0_SLICE_REQUIRED |		\
+		COMN_REQ_CIPHER0_SLICE_REQUIRED;
+#define COMN_REQ_CY1_ONLY(shram)			\
+		COMN_REQ_ORD_STRICT |			\
+		COMN_REQ_PTR_TYPE_FLAT |		\
+		(shram) |				\
+		COMN_REQ_PKE1_SLICE_REQUIRED |		\
+		COMN_REQ_AUTH1_SLICE_REQUIRED |		\
+		COMN_REQ_CIPHER1_SLICE_REQUIRED;
+
+#define COMN_RESP_CRYPTO_STATUS		__BIT(7)
+#define COMN_RESP_PKE_STATUS		__BIT(6)
+#define COMN_RESP_CMP_STATUS		__BIT(5)
+#define COMN_RESP_XLAT_STATUS		__BIT(4)
+#define COMN_RESP_PM_STATUS		__BIT(3)
+#define COMN_RESP_INIT_ADMIN_STATUS	__BIT(2)
+
+#define COMN_STATUS_FLAG_OK		0
+#define COMN_STATUS_FLAG_ERROR		1
+
+struct fw_la_ssl_tls_common {
+	uint8_t out_len;	/* Number of bytes of key material to output. */
+	uint8_t label_len;	/* Number of bytes of label for SSL and bytes
+				 * for TLS key generation  */
+};
+
+struct fw_la_mgf_common {
+	uint8_t hash_len;
+	/* Number of bytes of hash output by the QAT per iteration */
+	uint8_t seed_len;
+	/* Number of bytes of seed provided in src buffer for MGF1 */
+};
+
+struct fw_cipher_hdr {
+	uint8_t state_sz;
+	/* State size in quad words of the cipher algorithm used in this session.
+	 * Set to zero if the algorithm doesnt provide any state */
+	uint8_t offset;
+	/* Quad word offset from the content descriptor parameters address i.e.
+	 * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher
+	 * processing */
+	uint8_t curr_id;
+	/* Initialised with the cipher slice type */
+	uint8_t next_id;
+	/* Set to the next slice to pass the ciphered data through.
+	 * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+	 * anymore slices after cipher */
+	uint16_t resrvd;
+	/* Reserved padding byte to bring the struct to the word boundary. MUST be
+	 * set to 0 */
+	uint8_t state_padding_sz;
+	/* State padding size in quad words. Set to 0 if no padding is required. */
+	uint8_t key_sz;
+	/* Key size in quad words of the cipher algorithm used in this session */
+};
+
+struct fw_auth_hdr {
+	uint8_t hash_flags;
+	/* General flags defining the processing to perform. 0 is normal processing
+	 * and 1 means there is a nested hash processing loop to go through */
+	uint8_t offset;
+	/* Quad word offset from the content descriptor parameters address to the
+	 * parameters for the auth processing */
+	uint8_t curr_id;
+	/* Initialised with the auth slice type */
+	uint8_t next_id;
+	/* Set to the next slice to pass data through.
+	 * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+	 * anymore slices after auth */
+	union {
+		uint8_t inner_prefix_sz;
+		/* Size in bytes of the inner prefix data */
+		uint8_t aad_sz;
+		/* Size in bytes of padded AAD data to prefix to the packet for CCM
+		 *  or GCM processing */
+	} u;
+
+	uint8_t outer_prefix_sz;
+	/* Size in bytes of outer prefix data */
+	uint8_t final_sz;
+	/* Size in bytes of digest to be returned to the client if requested */
+	uint8_t inner_res_sz;
+	/* Size in bytes of the digest from the inner hash algorithm */
+	uint8_t resrvd;
+	/* This field is unused, assumed value is zero. */
+	uint8_t inner_state1_sz;
+	/* Size in bytes of inner hash state1 data. Must be a qword multiple */
+	uint8_t inner_state2_off;
+	/* Quad word offset from the content descriptor parameters pointer to the
+	 * inner state2 value */
+	uint8_t inner_state2_sz;
+	/* Size in bytes of inner hash state2 data. Must be a qword multiple */
+	uint8_t outer_config_off;
+	/* Quad word offset from the content descriptor parameters pointer to the
+	 * outer configuration information */
+	uint8_t outer_state1_sz;
+	/* Size in bytes of the outer state1 value */
+	uint8_t outer_res_sz;
+	/* Size in bytes of digest from the outer auth algorithm */
+	uint8_t outer_prefix_off;
+	/* Quad word offset from the start of the inner prefix data to the outer
+	 * prefix information. Should equal the rounded inner prefix size, converted
+	 * to qwords  */
+};
+
+#define FW_AUTH_HDR_FLAG_DO_NESTED	1
+#define FW_AUTH_HDR_FLAG_NO_NESTED	0
+
+struct fw_la_comn_req {
+	union {
+		uint16_t la_flags;
+		/* Definition of the common LA processing flags used for the
+		 * bulk processing */
+		union {
+			struct fw_la_ssl_tls_common ssl_tls_common;
+			/* For TLS or SSL Key Generation, this field is
+			 * overloaded with ssl_tls common information */
+			struct fw_la_mgf_common mgf_common;
+			/* For MGF Key Generation, this field is overloaded with
+			   mgf information */
+		} u;
+	} u;
+
+	union {
+		uint8_t resrvd;
+		/* If not useRd by a request this field must be set to 0 */
+		uint8_t tls_seed_len;
+		/* Byte Len of tls seed */
+		uint8_t req_params_blk_sz;
+		/* For bulk processing this field represents the request
+		 * parameters block size */
+		uint8_t trng_cfg_sz;
+		/* This field is used for TRNG_ENABLE requests to indicate the
+		 * size of the TRNG Slice configuration word. Size is in QW's */
+	} u1;
+	uint8_t la_cmd_id;
+	/* Definition of the LA command defined by this request */
+};
+
+#define LA_FLAGS_GCM_IV_LEN_FLAG	__BIT(9)
+#define LA_FLAGS_PROTO			__BITS(8, 6)
+#define LA_FLAGS_PROTO_SNOW_3G		__SHIFTIN(4, LA_FLAGS_PROTO)
+#define LA_FLAGS_PROTO_GCM		__SHIFTIN(2, LA_FLAGS_PROTO)
+#define LA_FLAGS_PROTO_CCM		__SHIFTIN(1, LA_FLAGS_PROTO)
+#define LA_FLAGS_PROTO_NO		__SHIFTIN(0, LA_FLAGS_PROTO)
+#define LA_FLAGS_DIGEST_IN_BUFFER	__BIT(5)
+#define LA_FLAGS_CMP_AUTH_RES		__BIT(4)
+#define LA_FLAGS_RET_AUTH_RES		__BIT(3)
+#define LA_FLAGS_UPDATE_STATE		__BIT(2)
+#define LA_FLAGS_PARTIAL		__BITS(1, 0)
+
+struct fw_la_bulk_req {
+	struct fw_comn_req_hdr comn_hdr;
+	/* Common request header */
+	uint32_t flow_id;
+	/* Field used by Firmware to limit the number of stateful requests
+	 * for a session being processed at a given point of time */
+	struct fw_la_comn_req comn_la_req;
+	/* Common LA request parameters */
+	struct fw_comn_req_mid comn_mid;
+	/* Common request middle section */
+	uint64_t req_params_addr;
+	/* Memory address of the request parameters */
+	union fw_comn_req_ftr comn_ftr;
+	/* Common request footer */
+};
+
+struct fw_la_resp {
+	struct fw_comn_resp_hdr comn_resp;
+	uint8_t resrvd[64 - sizeof(struct fw_comn_resp_hdr)];
+	/* FW_RESP_DEFAULT_SZ_HW15 */
+};
+
+struct fw_la_cipher_req_params {
+	uint8_t resrvd;
+	/* Reserved field and assumed set to 0 */
+	uint8_t cipher_state_sz;
+	/* Number of quad words of state data for the cipher algorithm */
+	uint8_t curr_id;
+	/* Initialised with the cipher slice type */
+	uint8_t next_id;
+	/* Set to the next slice to pass the ciphered data through.
+	 * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+	 * anymore slices after cipher */
+	uint16_t resrvd1;
+	/* Reserved field, should be set to zero*/
+	uint8_t resrvd2;
+	/* Reserved field, should be set to zero*/
+	uint8_t next_offset;
+	/* Offset in bytes to the next request parameter block */
+	uint32_t cipher_off;
+	/* Byte offset from the start of packet to the cipher data region */
+	uint32_t cipher_len;
+	/* Byte length of the cipher data region */
+	uint64_t state_address;
+	/* Flat buffer address in memory of the cipher state information. Unused
+	 * if the state size is 0 */
+};
+
+struct fw_la_auth_req_params {
+	uint8_t auth_res_sz;
+	/* Size in quad words of digest information to validate */
+	uint8_t hash_state_sz;
+	/* Number of quad words of inner and outer hash prefix data to process */
+	uint8_t curr_id;
+	/* Initialised with the auth slice type */
+	uint8_t next_id;
+	/* Set to the next slice to pass the auth data through.
+	 * Set to ICP_QAT_FW_SLICE_NULL for in-place auth-only requests
+	 * Set to ICP_QAT_FW_SLICE_DRAM_WR for all other request types
+	 * if the data is not to go through anymore slices after auth */
+	union {
+		uint16_t resrvd;
+		/* Reserved field should be set to zero for bulk services */
+		uint16_t tls_secret_len;
+		/* Length of Secret information for TLS.   */
+	} u;
+	uint8_t resrvd;
+	/* Reserved field, should be set to zero*/
+	uint8_t next_offset;
+	/* offset in bytes to the next request parameter block */
+	uint32_t auth_off;
+	/* Byte offset from the start of packet to the auth data region */
+	uint32_t auth_len;
+	/* Byte length of the auth data region */
+	union {
+		uint64_t prefix_addr;
+		/* Address of the prefix information */
+		uint64_t aad_addr;
+		/* Address of the AAD info in DRAM. Used for the CCM and GCM
+		 * protocols */
+	} u1;
+	uint64_t auth_res_address;
+	/* Address of the auth result information to validate or the location to
+	 * writeback the digest information to */
+};
+
+#endif
diff --git a/sys/dev/qat/qat_hw15var.h b/sys/dev/qat/qat_hw15var.h
new file mode 100644
index 000000000000..4939aba7ff12
--- /dev/null
+++ b/sys/dev/qat/qat_hw15var.h
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_hw15var.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2013 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_HW15VAR_H_
+#define _DEV_PCI_QAT_HW15VAR_H_
+
+CTASSERT(HASH_CONTENT_DESC_SIZE >=
+   sizeof(struct fw_auth_hdr) + MAX_HASH_SETUP_BLK_SZ);
+CTASSERT(CIPHER_CONTENT_DESC_SIZE >=
+   sizeof(struct fw_cipher_hdr) + MAX_CIPHER_SETUP_BLK_SZ);
+CTASSERT(CONTENT_DESC_MAX_SIZE >=
+    roundup(HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE,
+        QAT_OPTIMAL_ALIGN));
+CTASSERT(QAT_SYM_REQ_PARAMS_SIZE_PADDED >=
+    roundup(sizeof(struct fw_la_cipher_req_params) +
+        sizeof(struct fw_la_auth_req_params), QAT_OPTIMAL_ALIGN));
+
+/* length of the 5 long words of the request that are stored in the session
+ * This is rounded up to 32 in order to use the fast memcopy function */
+#define QAT_HW15_SESSION_REQ_CACHE_SIZE	(32)
+
+void		qat_msg_req_type_populate(struct arch_if_req_hdr *,
+		    enum arch_if_req, uint32_t);
+void		qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *, bus_addr_t,
+		    uint8_t, uint8_t, uint16_t, uint32_t);
+void		qat_msg_service_cmd_populate(struct fw_la_bulk_req *,
+		    enum fw_la_cmd_id, uint16_t);
+void		qat_msg_cmn_mid_populate(struct fw_comn_req_mid *, void *,
+		    uint64_t , uint64_t);
+void		qat_msg_req_params_populate(struct fw_la_bulk_req *, bus_addr_t,
+		    uint8_t);
+void		qat_msg_cmn_footer_populate(union fw_comn_req_ftr *, uint64_t);
+void		qat_msg_params_populate(struct fw_la_bulk_req *,
+		    struct qat_crypto_desc *, uint8_t, uint16_t,
+		    uint16_t);
+
+
+int		qat_adm_ring_init(struct qat_softc *);
+int		qat_adm_ring_send_init(struct qat_softc *);
+
+void		qat_hw15_crypto_setup_desc(struct qat_crypto *,
+		    struct qat_session *, struct qat_crypto_desc *);
+void		qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *,
+		    struct qat_session *, struct qat_crypto_desc const *,
+		    struct qat_sym_cookie *, struct cryptop *);
+
+#endif
diff --git a/sys/dev/qat/qat_hw17.c b/sys/dev/qat/qat_hw17.c
new file mode 100644
index 000000000000..643b624ba840
--- /dev/null
+++ b/sys/dev/qat/qat_hw17.c
@@ -0,0 +1,662 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+#if 0
+__KERNEL_RCSID(0, "$NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/proc.h>
+
+#include <machine/bus.h>
+
+#include <opencrypto/xform.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include "qatreg.h"
+#include "qat_hw17reg.h"
+#include "qatvar.h"
+#include "qat_hw17var.h"
+
+int		qat_adm_mailbox_put_msg_sync(struct qat_softc *, uint32_t,
+		    void *, void *);
+int		qat_adm_mailbox_send(struct qat_softc *,
+		    struct fw_init_admin_req *, struct fw_init_admin_resp *);
+int		qat_adm_mailbox_send_init_me(struct qat_softc *);
+int		qat_adm_mailbox_send_hb_timer(struct qat_softc *);
+int		qat_adm_mailbox_send_fw_status(struct qat_softc *);
+int		qat_adm_mailbox_send_constants(struct qat_softc *);
+
+int
+qat_adm_mailbox_init(struct qat_softc *sc)
+{
+	uint64_t addr;
+	int error;
+	struct qat_dmamem *qdm;
+
+	error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_dma, 1,
+	    PAGE_SIZE, PAGE_SIZE);
+	if (error)
+		return error;
+
+	qdm = &sc->sc_admin_comms.qadc_const_tbl_dma;
+	error = qat_alloc_dmamem(sc, qdm, 1, PAGE_SIZE, PAGE_SIZE);
+	if (error)
+		return error;
+
+	memcpy(qdm->qdm_dma_vaddr,
+	    mailbox_const_tab, sizeof(mailbox_const_tab));
+
+	bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map,
+	    BUS_DMASYNC_PREWRITE);
+
+	error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_hb_dma, 1,
+	    PAGE_SIZE, PAGE_SIZE);
+	if (error)
+		return error;
+
+	addr = (uint64_t)sc->sc_admin_comms.qadc_dma.qdm_dma_seg.ds_addr;
+	qat_misc_write_4(sc, ADMINMSGUR, addr >> 32);
+	qat_misc_write_4(sc, ADMINMSGLR, addr);
+
+	return 0;
+}
+
+int
+qat_adm_mailbox_put_msg_sync(struct qat_softc *sc, uint32_t ae,
+    void *in, void *out)
+{
+	struct qat_dmamem *qdm;
+	uint32_t mailbox;
+	bus_size_t mb_offset = MAILBOX_BASE + (ae * MAILBOX_STRIDE);
+	int offset = ae * ADMINMSG_LEN * 2;
+	int times, received;
+	uint8_t *buf = (uint8_t *)sc->sc_admin_comms.qadc_dma.qdm_dma_vaddr + offset;
+
+	mailbox = qat_misc_read_4(sc, mb_offset);
+	if (mailbox == 1)
+		return EAGAIN;
+
+	qdm = &sc->sc_admin_comms.qadc_dma;
+	memcpy(buf, in, ADMINMSG_LEN);
+	bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map,
+	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	qat_misc_write_4(sc, mb_offset, 1);
+
+	received = 0;
+	for (times = 0; times < 50; times++) {
+		DELAY(20000);
+		if (qat_misc_read_4(sc, mb_offset) == 0) {
+			received = 1;
+			break;
+		}
+	}
+	if (received) {
+		bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map,
+		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+		memcpy(out, buf + ADMINMSG_LEN, ADMINMSG_LEN);
+	} else {
+		device_printf(sc->sc_dev,
+		    "Failed to send admin msg to accelerator\n");
+	}
+
+	return received ? 0 : EFAULT;
+}
+
+int
+qat_adm_mailbox_send(struct qat_softc *sc,
+    struct fw_init_admin_req *req, struct fw_init_admin_resp *resp)
+{
+	int error;
+	uint32_t mask;
+	uint8_t ae;
+
+	for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+
+		error = qat_adm_mailbox_put_msg_sync(sc, ae, req, resp);
+		if (error)
+			return error;
+		if (resp->init_resp_hdr.status) {
+			device_printf(sc->sc_dev,
+			    "Failed to send admin msg: cmd %d\n",
+			    req->init_admin_cmd_id);
+			return EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+int
+qat_adm_mailbox_send_init_me(struct qat_softc *sc)
+{
+	struct fw_init_admin_req req;
+	struct fw_init_admin_resp resp;
+
+	memset(&req, 0, sizeof(req));
+	req.init_admin_cmd_id = FW_INIT_ME;
+
+	return qat_adm_mailbox_send(sc, &req, &resp);
+}
+
+int
+qat_adm_mailbox_send_hb_timer(struct qat_softc *sc)
+{
+	struct fw_init_admin_req req;
+	struct fw_init_admin_resp resp;
+
+	memset(&req, 0, sizeof(req));
+	req.init_admin_cmd_id = FW_HEARTBEAT_TIMER_SET;
+
+	req.init_cfg_ptr = sc->sc_admin_comms.qadc_hb_dma.qdm_dma_seg.ds_addr;
+	req.heartbeat_ticks =
+	    sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_HB_INTERVAL;
+
+	return qat_adm_mailbox_send(sc, &req, &resp);
+}
+
+int
+qat_adm_mailbox_send_fw_status(struct qat_softc *sc)
+{
+	int error;
+	struct fw_init_admin_req req;
+	struct fw_init_admin_resp resp;
+
+	memset(&req, 0, sizeof(req));
+	req.init_admin_cmd_id = FW_STATUS_GET;
+
+	error = qat_adm_mailbox_send(sc, &req, &resp);
+	if (error)
+		return error;
+
+	return 0;
+}
+
+int
+qat_adm_mailbox_send_constants(struct qat_softc *sc)
+{
+	struct fw_init_admin_req req;
+	struct fw_init_admin_resp resp;
+
+	memset(&req, 0, sizeof(req));
+	req.init_admin_cmd_id = FW_CONSTANTS_CFG;
+
+	req.init_cfg_sz = 1024;
+	req.init_cfg_ptr =
+	    sc->sc_admin_comms.qadc_const_tbl_dma.qdm_dma_seg.ds_addr;
+
+	return qat_adm_mailbox_send(sc, &req, &resp);
+}
+
+int
+qat_adm_mailbox_send_init(struct qat_softc *sc)
+{
+	int error;
+
+	error = qat_adm_mailbox_send_init_me(sc);
+	if (error)
+		return error;
+
+	error = qat_adm_mailbox_send_hb_timer(sc);
+	if (error)
+		return error;
+
+	error = qat_adm_mailbox_send_fw_status(sc);
+	if (error)
+		return error;
+
+	return qat_adm_mailbox_send_constants(sc);
+}
+
+int
+qat_arb_init(struct qat_softc *sc)
+{
+	uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1;
+	uint32_t arb, i;
+	const uint32_t *thd_2_arb_cfg;
+
+	/* Service arb configured for 32 bytes responses and
+	 * ring flow control check enabled. */
+	for (arb = 0; arb < MAX_ARB; arb++)
+		qat_arb_sarconfig_write_4(sc, arb, arb_cfg);
+
+	/* Map worker threads to service arbiters */
+	sc->sc_hw.qhw_get_arb_mapping(sc, &thd_2_arb_cfg);
+
+	if (!thd_2_arb_cfg)
+		return EINVAL;
+
+	for (i = 0; i < sc->sc_hw.qhw_num_engines; i++)
+		qat_arb_wrk_2_ser_map_write_4(sc, i, *(thd_2_arb_cfg + i));
+
+	return 0;
+}
+
+int
+qat_set_ssm_wdtimer(struct qat_softc *sc)
+{
+	uint32_t timer;
+	u_int mask;
+	int i;
+
+	timer = sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_SSM_WDT;
+	for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) {
+		if (!(mask & 1))
+			continue;
+		qat_misc_write_4(sc, SSMWDT(i), timer);
+		qat_misc_write_4(sc, SSMWDTPKE(i), timer);
+	}
+
+	return 0;
+}
+
+int
+qat_check_slice_hang(struct qat_softc *sc)
+{
+	int handled = 0;
+
+	return handled;
+}
+
+static uint32_t
+qat_hw17_crypto_setup_cipher_ctrl(struct qat_crypto_desc *desc,
+    struct qat_session *qs, uint32_t cd_blk_offset,
+    struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice)
+{
+	struct fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl =
+	    (struct fw_cipher_cd_ctrl_hdr *)&req_tmpl->cd_ctrl;
+
+	desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ;
+	desc->qcd_cipher_offset = cd_blk_offset;
+
+	cipher_cd_ctrl->cipher_state_sz = desc->qcd_cipher_blk_sz >> 3;
+	cipher_cd_ctrl->cipher_key_sz = qs->qs_cipher_klen >> 3;
+	cipher_cd_ctrl->cipher_cfg_offset = cd_blk_offset >> 3;
+	FW_COMN_CURR_ID_SET(cipher_cd_ctrl, FW_SLICE_CIPHER);
+	FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, next_slice);
+
+	return roundup(sizeof(struct hw_cipher_config) + qs->qs_cipher_klen, 8);
+}
+
+static void
+qat_hw17_crypto_setup_cipher_cdesc(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, const struct cryptop *crp,
+    union hw_cipher_algo_blk *cipher)
+{
+	const uint8_t *key;
+
+	cipher->max.cipher_config.val =
+	    qat_crypto_load_cipher_session(desc, qs);
+	if (crp != NULL && crp->crp_cipher_key != NULL)
+		key = crp->crp_cipher_key;
+	else
+		key = qs->qs_cipher_key;
+	memcpy(cipher->max.key, key, qs->qs_cipher_klen);
+}
+
+static uint32_t
+qat_hw17_crypto_setup_auth_ctrl(struct qat_crypto_desc *desc,
+    struct qat_session *qs, uint32_t cd_blk_offset,
+    struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice)
+{
+	struct fw_auth_cd_ctrl_hdr *auth_cd_ctrl =
+	    (struct fw_auth_cd_ctrl_hdr *)&req_tmpl->cd_ctrl;
+	struct qat_sym_hash_def const *hash_def;
+
+	(void)qat_crypto_load_auth_session(desc, qs, &hash_def);
+
+	auth_cd_ctrl->hash_cfg_offset = cd_blk_offset >> 3;
+	auth_cd_ctrl->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED;
+	auth_cd_ctrl->inner_res_sz = hash_def->qshd_alg->qshai_digest_len;
+	auth_cd_ctrl->final_sz = hash_def->qshd_alg->qshai_sah->hashsize;
+
+	auth_cd_ctrl->inner_state1_sz =
+	    roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
+	auth_cd_ctrl->inner_state2_sz =
+	    roundup(hash_def->qshd_qat->qshqi_state2_len, 8);
+	auth_cd_ctrl->inner_state2_offset =
+	    auth_cd_ctrl->hash_cfg_offset +
+	    ((sizeof(struct hw_auth_setup) +
+	    auth_cd_ctrl->inner_state1_sz) >> 3);
+
+	FW_COMN_CURR_ID_SET(auth_cd_ctrl, FW_SLICE_AUTH);
+	FW_COMN_NEXT_ID_SET(auth_cd_ctrl, next_slice);
+
+	desc->qcd_auth_sz = auth_cd_ctrl->final_sz;
+	desc->qcd_auth_offset = cd_blk_offset;
+	desc->qcd_gcm_aad_sz_offset1 =
+	    cd_blk_offset + offsetof(union hw_auth_algo_blk, max.state1) +
+	    auth_cd_ctrl->inner_state1_sz + AES_BLOCK_LEN;
+
+	return roundup(auth_cd_ctrl->inner_state1_sz +
+	    auth_cd_ctrl->inner_state2_sz +
+	    sizeof(struct hw_auth_setup), 8);
+}
+
+static void
+qat_hw17_crypto_setup_auth_cdesc(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, const struct cryptop *crp,
+    union hw_auth_algo_blk *auth)
+{
+	struct qat_sym_hash_def const *hash_def;
+	uint8_t inner_state1_sz, *state1, *state2;
+	const uint8_t *key;
+
+	auth->max.inner_setup.auth_config.config =
+	    qat_crypto_load_auth_session(desc, qs, &hash_def);
+	auth->max.inner_setup.auth_counter.counter =
+	    htobe32(hash_def->qshd_qat->qshqi_auth_counter);
+	inner_state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
+
+	state1 = auth->max.state1;
+	state2 = auth->max.state1 + inner_state1_sz;
+	switch (qs->qs_auth_algo) {
+	case HW_AUTH_ALGO_GALOIS_128:
+		key = NULL;
+		if (crp != NULL && crp->crp_cipher_key != NULL)
+			key = crp->crp_cipher_key;
+		else if (qs->qs_cipher_key != NULL)
+			key = qs->qs_cipher_key;
+		if (key != NULL) {
+			qat_crypto_gmac_precompute(desc, key,
+			    qs->qs_cipher_klen, hash_def, state2);
+		}
+		break;
+	case HW_AUTH_ALGO_SHA1:
+	case HW_AUTH_ALGO_SHA256:
+	case HW_AUTH_ALGO_SHA384:
+	case HW_AUTH_ALGO_SHA512:
+		switch (qs->qs_auth_mode) {
+		case HW_AUTH_MODE0:
+			memcpy(state1, hash_def->qshd_alg->qshai_init_state,
+			    inner_state1_sz);
+			/* Override for mode 0 hashes. */
+			auth->max.inner_setup.auth_counter.counter = 0;
+			break;
+		case HW_AUTH_MODE1:
+			if (crp != NULL && crp->crp_auth_key != NULL)
+				key = crp->crp_auth_key;
+			else
+				key = qs->qs_auth_key;
+			if (key != NULL) {
+				qat_crypto_hmac_precompute(desc, key,
+				    qs->qs_auth_klen, hash_def, state1, state2);
+			}
+			break;
+		default:
+			panic("%s: unhandled auth mode %d", __func__,
+			    qs->qs_auth_mode);
+		}
+		break;
+	default:
+		panic("%s: unhandled auth algorithm %d", __func__,
+		    qs->qs_auth_algo);
+	}
+}
+
+static void
+qat_hw17_init_comn_req_hdr(struct qat_crypto_desc *desc,
+    struct fw_la_bulk_req *req)
+{
+	union fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+	struct fw_comn_req_hdr *req_hdr = &req->comn_hdr;
+
+	req_hdr->service_cmd_id = desc->qcd_cmd_id;
+	req_hdr->hdr_flags = FW_COMN_VALID;
+	req_hdr->service_type = FW_COMN_REQ_CPM_FW_LA;
+	req_hdr->comn_req_flags = FW_COMN_FLAGS_BUILD(
+	    COMN_CD_FLD_TYPE_64BIT_ADR, COMN_PTR_TYPE_SGL);
+	req_hdr->serv_specif_flags = 0;
+	cd_pars->s.content_desc_addr = desc->qcd_desc_paddr;
+}
+
+void
+qat_hw17_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs,
+    struct qat_crypto_desc *desc)
+{
+	union hw_cipher_algo_blk *cipher;
+	union hw_auth_algo_blk *auth;
+	struct fw_la_bulk_req *req_tmpl;
+	struct fw_comn_req_hdr *req_hdr;
+	uint32_t cd_blk_offset = 0;
+	int i;
+	uint8_t *cd_blk_ptr;
+
+	req_tmpl = (struct fw_la_bulk_req *)desc->qcd_req_cache;
+	req_hdr = &req_tmpl->comn_hdr;
+	cd_blk_ptr = desc->qcd_content_desc;
+
+	memset(req_tmpl, 0, sizeof(struct fw_la_bulk_req));
+	qat_hw17_init_comn_req_hdr(desc, req_tmpl);
+
+	for (i = 0; i < MAX_FW_SLICE; i++) {
+		switch (desc->qcd_slices[i]) {
+		case FW_SLICE_CIPHER:
+			cipher = (union hw_cipher_algo_blk *)(cd_blk_ptr +
+			    cd_blk_offset);
+			cd_blk_offset += qat_hw17_crypto_setup_cipher_ctrl(desc,
+			    qs, cd_blk_offset, req_tmpl,
+			    desc->qcd_slices[i + 1]);
+			qat_hw17_crypto_setup_cipher_cdesc(desc, qs, NULL,
+			    cipher);
+			break;
+		case FW_SLICE_AUTH:
+			auth = (union hw_auth_algo_blk *)(cd_blk_ptr +
+			    cd_blk_offset);
+			cd_blk_offset += qat_hw17_crypto_setup_auth_ctrl(desc,
+			    qs, cd_blk_offset, req_tmpl,
+			    desc->qcd_slices[i + 1]);
+			qat_hw17_crypto_setup_auth_cdesc(desc, qs, NULL, auth);
+			req_hdr->serv_specif_flags |= FW_LA_RET_AUTH_RES;
+			break;
+		case FW_SLICE_DRAM_WR:
+			i = MAX_FW_SLICE; /* end of chain */
+			break;
+		default:
+			MPASS(0);
+			break;
+		}
+	}
+
+	req_tmpl->cd_pars.s.content_desc_params_sz =
+	    roundup(cd_blk_offset, QAT_OPTIMAL_ALIGN) >> 3;
+	if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128)
+		req_hdr->serv_specif_flags |=
+		    FW_LA_PROTO_GCM | FW_LA_GCM_IV_LEN_12_OCTETS;
+
+	bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
+	    qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE);
+}
+
+static void
+qat_hw17_crypto_req_setkey(const struct qat_crypto_desc *desc,
+    const struct qat_session *qs, struct qat_sym_cookie *qsc,
+    struct fw_la_bulk_req *bulk_req, const struct cryptop *crp)
+{
+	union hw_auth_algo_blk *auth;
+	union hw_cipher_algo_blk *cipher;
+	uint8_t *cdesc;
+	int i;
+
+	cdesc = qsc->qsc_content_desc;
+	memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE);
+	for (i = 0; i < MAX_FW_SLICE; i++) {
+		switch (desc->qcd_slices[i]) {
+		case FW_SLICE_CIPHER:
+			cipher = (union hw_cipher_algo_blk *)
+			    (cdesc + desc->qcd_cipher_offset);
+			qat_hw17_crypto_setup_cipher_cdesc(desc, qs, crp,
+			    cipher);
+			break;
+		case FW_SLICE_AUTH:
+			auth = (union hw_auth_algo_blk *)
+			    (cdesc + desc->qcd_auth_offset);
+			qat_hw17_crypto_setup_auth_cdesc(desc, qs, crp, auth);
+			break;
+		case FW_SLICE_DRAM_WR:
+			i = MAX_FW_SLICE; /* end of chain */
+			break;
+		default:
+			MPASS(0);
+		}
+	}
+
+	bulk_req->cd_pars.s.content_desc_addr = qsc->qsc_content_desc_paddr;
+}
+
+void
+qat_hw17_crypto_setup_req_params(struct qat_crypto_bank *qcb __unused,
+    struct qat_session *qs, const struct qat_crypto_desc *desc,
+    struct qat_sym_cookie *qsc, struct cryptop *crp)
+{
+	struct qat_sym_bulk_cookie *qsbc;
+	struct fw_la_bulk_req *bulk_req;
+	struct fw_la_cipher_req_params *cipher_param;
+	struct fw_la_auth_req_params *auth_param;
+	bus_addr_t digest_paddr;
+	uint32_t aad_sz, *aad_szp;
+	uint8_t *req_params_ptr;
+	enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id;
+
+	qsbc = &qsc->u.qsc_bulk_cookie;
+	bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg;
+
+	memcpy(bulk_req, desc->qcd_req_cache, sizeof(struct fw_la_bulk_req));
+	bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc;
+	bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr;
+	bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr;
+	if (__predict_false(crp->crp_cipher_key != NULL ||
+	    crp->crp_auth_key != NULL))
+		qat_hw17_crypto_req_setkey(desc, qs, qsc, bulk_req, crp);
+
+	digest_paddr = 0;
+	if (desc->qcd_auth_sz != 0)
+		digest_paddr = qsc->qsc_auth_res_paddr;
+
+	req_params_ptr = (uint8_t *)&bulk_req->serv_specif_rqpars;
+	cipher_param = (struct fw_la_cipher_req_params *)req_params_ptr;
+	auth_param = (struct fw_la_auth_req_params *)
+	    (req_params_ptr + sizeof(struct fw_la_cipher_req_params));
+
+	cipher_param->u.s.cipher_IV_ptr = qsc->qsc_iv_buf_paddr;
+
+	/*
+	 * The SG list layout is a bit different for GCM and GMAC, it's simpler
+	 * to handle those cases separately.
+	 */
+	if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
+		if (cmd_id != FW_LA_CMD_AUTH) {
+			/*
+			 * Don't fill out the cipher block if we're doing GMAC
+			 * only.
+			 */
+			cipher_param->cipher_offset = 0;
+			cipher_param->cipher_length = crp->crp_payload_length;
+		}
+		auth_param->auth_off = 0;
+		auth_param->auth_len = crp->crp_payload_length;
+		auth_param->auth_res_addr = digest_paddr;
+		auth_param->auth_res_sz = desc->qcd_auth_sz;
+		auth_param->u1.aad_adr =
+		    crp->crp_aad_length > 0 ? qsc->qsc_gcm_aad_paddr : 0;
+		auth_param->u2.aad_sz =
+		    roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN);
+		auth_param->hash_state_sz = auth_param->u2.aad_sz >> 3;
+
+		/*
+		 * Update the hash state block if necessary.  This only occurs
+		 * when the AAD length changes between requests in a session and
+		 * is synchronized by qat_process().
+		 */
+		aad_sz = htobe32(crp->crp_aad_length);
+		aad_szp = (uint32_t *)(
+		    __DECONST(uint8_t *, desc->qcd_content_desc) +
+		    desc->qcd_gcm_aad_sz_offset1);
+		if (__predict_false(*aad_szp != aad_sz)) {
+			*aad_szp = aad_sz;
+			bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
+			    qs->qs_desc_mem.qdm_dma_map,
+			    BUS_DMASYNC_PREWRITE);
+		}
+	} else {
+		if (cmd_id != FW_LA_CMD_AUTH) {
+			cipher_param->cipher_offset =
+			    crp->crp_aad_length == 0 ? 0 :
+			    crp->crp_payload_start - crp->crp_aad_start;
+			cipher_param->cipher_length = crp->crp_payload_length;
+		}
+		if (cmd_id != FW_LA_CMD_CIPHER) {
+			auth_param->auth_off = 0;
+			auth_param->auth_len =
+			    crp->crp_payload_length + crp->crp_aad_length;
+			auth_param->auth_res_addr = digest_paddr;
+			auth_param->auth_res_sz = desc->qcd_auth_sz;
+			auth_param->u1.aad_adr = 0;
+			auth_param->u2.aad_sz = 0;
+			auth_param->hash_state_sz = 0;
+		}
+	}
+}
diff --git a/sys/dev/qat/qat_hw17reg.h b/sys/dev/qat/qat_hw17reg.h
new file mode 100644
index 000000000000..6655b905dde0
--- /dev/null
+++ b/sys/dev/qat/qat_hw17reg.h
@@ -0,0 +1,2460 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_hw17reg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_HW17REG_H_
+#define _DEV_PCI_QAT_HW17REG_H_
+
+/* Default message size in bytes */
+#define FW_REQ_DEFAULT_SZ_HW17		128
+#define FW_RESP_DEFAULT_SZ_HW17		32
+
+/* -------------------------------------------------------------------------- */
+/* accel */
+
+enum fw_init_admin_cmd_id {
+	FW_INIT_ME = 0,
+	FW_TRNG_ENABLE = 1,
+	FW_TRNG_DISABLE = 2,
+	FW_CONSTANTS_CFG = 3,
+	FW_STATUS_GET = 4,
+	FW_COUNTERS_GET = 5,
+	FW_LOOPBACK = 6,
+	FW_HEARTBEAT_SYNC = 7,
+	FW_HEARTBEAT_GET = 8,
+	FW_COMP_CAPABILITY_GET = 9,
+	FW_CRYPTO_CAPABILITY_GET = 10,
+	FW_HEARTBEAT_TIMER_SET = 13,
+};
+
+enum fw_init_admin_resp_status {
+	FW_INIT_RESP_STATUS_SUCCESS = 0,
+	FW_INIT_RESP_STATUS_FAIL = 1,
+	FW_INIT_RESP_STATUS_UNSUPPORTED = 4
+};
+
+struct fw_init_admin_req {
+	uint16_t init_cfg_sz;
+	uint8_t resrvd1;
+	uint8_t init_admin_cmd_id;
+	uint32_t resrvd2;
+	uint64_t opaque_data;
+	uint64_t init_cfg_ptr;
+
+	union {
+		struct {
+			uint16_t ibuf_size_in_kb;
+			uint16_t resrvd3;
+		};
+		uint32_t heartbeat_ticks;
+	};
+
+	uint32_t resrvd4;
+};
+
+struct fw_init_admin_resp_hdr {
+	uint8_t flags;
+	uint8_t resrvd1;
+	uint8_t status;
+	uint8_t init_admin_cmd_id;
+};
+
+enum fw_init_admin_init_flag {
+	FW_INIT_FLAG_PKE_DISABLED = 0
+};
+
+struct fw_init_admin_fw_capability_resp_hdr {
+	uint16_t     reserved;
+	uint8_t     status;
+	uint8_t     init_admin_cmd_id;
+};
+
+struct fw_init_admin_capability_resp {
+	struct fw_init_admin_fw_capability_resp_hdr init_resp_hdr;
+	uint32_t extended_features;
+	uint64_t opaque_data;
+	union {
+		struct {
+			uint16_t    compression_algos;
+			uint16_t    checksum_algos;
+			uint32_t    deflate_capabilities;
+			uint32_t    resrvd1;
+			uint32_t    lzs_capabilities;
+		} compression;
+		struct {
+			uint32_t    cipher_algos;
+			uint32_t    hash_algos;
+			uint16_t    keygen_algos;
+			uint16_t    other;
+			uint16_t    public_key_algos;
+			uint16_t    prime_algos;
+		} crypto;
+	};
+};
+
+struct fw_init_admin_resp_pars {
+	union {
+		uint32_t resrvd1[4];
+		struct {
+			uint32_t version_patch_num;
+			uint8_t context_id;
+			uint8_t ae_id;
+			uint16_t resrvd1;
+			uint64_t resrvd2;
+		} s1;
+		struct {
+			uint64_t req_rec_count;
+			uint64_t resp_sent_count;
+		} s2;
+	} u;
+};
+
+struct fw_init_admin_hb_cnt {
+	uint16_t resp_heartbeat_cnt;
+	uint16_t req_heartbeat_cnt;
+};
+
+#define QAT_NUM_THREADS 8
+
+struct fw_init_admin_hb_stats {
+	struct fw_init_admin_hb_cnt stats[QAT_NUM_THREADS];
+};
+
+struct fw_init_admin_resp {
+	struct fw_init_admin_resp_hdr init_resp_hdr;
+	union {
+		uint32_t resrvd2;
+		struct {
+			uint16_t version_minor_num;
+			uint16_t version_major_num;
+		} s;
+	} u;
+	uint64_t opaque_data;
+	struct fw_init_admin_resp_pars init_resp_pars;
+};
+
+#define FW_COMN_HEARTBEAT_OK 0
+#define FW_COMN_HEARTBEAT_BLOCKED 1
+#define FW_COMN_HEARTBEAT_FLAG_BITPOS 0
+#define FW_COMN_HEARTBEAT_FLAG_MASK 0x1
+#define FW_COMN_STATUS_RESRVD_FLD_MASK 0xFE
+#define FW_COMN_HEARTBEAT_HDR_FLAG_GET(hdr_t) \
+	FW_COMN_HEARTBEAT_FLAG_GET(hdr_t.flags)
+
+#define FW_COMN_HEARTBEAT_HDR_FLAG_SET(hdr_t, val) \
+	FW_COMN_HEARTBEAT_FLAG_SET(hdr_t, val)
+
+#define FW_COMN_HEARTBEAT_FLAG_GET(flags) \
+	QAT_FIELD_GET(flags, \
+		 FW_COMN_HEARTBEAT_FLAG_BITPOS, \
+		 FW_COMN_HEARTBEAT_FLAG_MASK)
+
+/* -------------------------------------------------------------------------- */
+
+/* Big assumptions that both bitpos and mask are constants */
+#define FIELD_SET(flags, val, bitpos, mask)                                \
+    (flags) =                                                                  \
+        (((flags) & (~((mask) << (bitpos)))) | (((val) & (mask)) << (bitpos)))
+
+#define FIELD_GET(flags, bitpos, mask) (((flags) >> (bitpos)) & (mask))
+
+#define FLAG_SET(flags, bitpos) (flags) = ((flags) | (1 << (bitpos)))
+
+#define FLAG_CLEAR(flags, bitpos) (flags) = ((flags) & (~(1 << (bitpos))))
+
+#define FLAG_GET(flags, bitpos) (((flags) >> (bitpos)) & 1)
+
+/* Default request and response ring size in bytes  */
+#define FW_REQ_DEFAULT_SZ 128
+#define FW_RESP_DEFAULT_SZ 32
+
+#define FW_COMN_ONE_BYTE_SHIFT 8
+#define FW_COMN_SINGLE_BYTE_MASK 0xFF
+
+/* Common Request - Block sizes definitions in multiples of individual long
+ * words  */
+#define FW_NUM_LONGWORDS_1 1
+#define FW_NUM_LONGWORDS_2 2
+#define FW_NUM_LONGWORDS_3 3
+#define FW_NUM_LONGWORDS_4 4
+#define FW_NUM_LONGWORDS_5 5
+#define FW_NUM_LONGWORDS_6 6
+#define FW_NUM_LONGWORDS_7 7
+#define FW_NUM_LONGWORDS_10 10
+#define FW_NUM_LONGWORDS_13 13
+
+/* Definition of the associated service Id for NULL service type.
+   Note: the response is expected to use FW_COMN_RESP_SERV_CPM_FW  */
+#define FW_NULL_REQ_SERV_ID 1
+
+/*
+ * Definition of the firmware interface service users, for
+ * responses.
+ * Enumeration which is used to indicate the ids of the services
+ * for responses using the external firmware interfaces.
+ */
+
+enum fw_comn_resp_serv_id {
+	FW_COMN_RESP_SERV_NULL,     /* NULL service id type */
+	FW_COMN_RESP_SERV_CPM_FW,   /* CPM FW Service ID */
+	FW_COMN_RESP_SERV_DELIMITER /* Delimiter service id type */
+};
+
+/*
+ * Definition of the request types
+ * Enumeration which is used to indicate the ids of the request
+ * types used in each of the external firmware interfaces
+ */
+
+enum fw_comn_request_id {
+	FW_COMN_REQ_NULL = 0,        /* NULL request type */
+	FW_COMN_REQ_CPM_FW_PKE = 3,  /* CPM FW PKE Request */
+	FW_COMN_REQ_CPM_FW_LA = 4,   /* CPM FW Lookaside Request */
+	FW_COMN_REQ_CPM_FW_DMA = 7,  /* CPM FW DMA Request */
+	FW_COMN_REQ_CPM_FW_COMP = 9, /* CPM FW Compression Request */
+	FW_COMN_REQ_DELIMITER        /* End delimiter */
+
+};
+
+/*
+ * Definition of the common QAT FW request content descriptor field -
+ * points to the content descriptor parameters or itself contains service-
+ * specific data. Also specifies content descriptor parameter size.
+ * Contains reserved fields.
+ * Common section of the request used across all of the services exposed
+ * by the QAT FW. Each of the services inherit these common fields
+ */
+union fw_comn_req_hdr_cd_pars {
+	/* LWs 2-5 */
+	struct
+	{
+		uint64_t content_desc_addr;
+		/* Address of the content descriptor */
+
+		uint16_t content_desc_resrvd1;
+		/* Content descriptor reserved field */
+
+		uint8_t content_desc_params_sz;
+		/* Size of the content descriptor parameters in quad words. These
+		 * parameters describe the session setup configuration info for the
+		 * slices that this request relies upon i.e. the configuration word and
+		 * cipher key needed by the cipher slice if there is a request for
+		 * cipher processing. */
+
+		uint8_t content_desc_hdr_resrvd2;
+		/* Content descriptor reserved field */
+
+		uint32_t content_desc_resrvd3;
+		/* Content descriptor reserved field */
+	} s;
+
+	struct
+	{
+		uint32_t serv_specif_fields[FW_NUM_LONGWORDS_4];
+
+	} s1;
+
+};
+
+/*
+ * Definition of the common QAT FW request middle block.
+ * Common section of the request used across all of the services exposed
+ * by the QAT FW. Each of the services inherit these common fields
+ */
+struct fw_comn_req_mid
+{
+	/* LWs 6-13 */
+	uint64_t opaque_data;
+	/* Opaque data passed unmodified from the request to response messages by
+	 * firmware (fw) */
+
+	uint64_t src_data_addr;
+	/* Generic definition of the source data supplied to the QAT AE. The
+	 * common flags are used to further describe the attributes of this
+	 * field */
+
+	uint64_t dest_data_addr;
+	/* Generic definition of the destination data supplied to the QAT AE. The
+	 * common flags are used to further describe the attributes of this
+	 * field */
+
+	uint32_t src_length;
+	/* Length of source flat buffer incase src buffer
+	 * type is flat */
+
+	uint32_t dst_length;
+	/* Length of source flat buffer incase dst buffer
+	 * type is flat */
+
+};
+
+/*
+ * Definition of the common QAT FW request content descriptor control
+ * block.
+ *
+ * Service specific section of the request used across all of the services
+ * exposed by the QAT FW. Each of the services populates this block
+ * uniquely. Refer to the service-specific header structures e.g.
+ * 'fw_cipher_hdr_s' (for Cipher) etc.
+ */
+struct fw_comn_req_cd_ctrl
+{
+	/* LWs 27-31 */
+	uint32_t content_desc_ctrl_lw[FW_NUM_LONGWORDS_5];
+
+};
+
+/*
+ * Definition of the common QAT FW request header.
+ * Common section of the request used across all of the services exposed
+ * by the QAT FW. Each of the services inherit these common fields. The
+ * reserved field of 7 bits and the service command Id field are all
+ * service-specific fields, along with the service specific flags.
+ */
+struct fw_comn_req_hdr
+{
+	/* LW0 */
+	uint8_t resrvd1;
+	/* reserved field */
+
+	uint8_t service_cmd_id;
+	/* Service Command Id  - this field is service-specific
+	 * Please use service-specific command Id here e.g.Crypto Command Id
+	 * or Compression Command Id etc. */
+
+	uint8_t service_type;
+	/* Service type */
+
+	uint8_t hdr_flags;
+	/* This represents a flags field for the Service Request.
+	 * The most significant bit is the 'valid' flag and the only
+	 * one used. All remaining bit positions are unused and
+	 * are therefore reserved and need to be set to 0. */
+
+	/* LW1 */
+	uint16_t serv_specif_flags;
+	/* Common Request service-specific flags
+	 * e.g. Symmetric Crypto Command Flags */
+
+	uint16_t comn_req_flags;
+	/* Common Request Flags consisting of
+	 * - 14 reserved bits,
+	 * - 1 Content Descriptor field type bit and
+	 * - 1 Source/destination pointer type bit */
+
+};
+
+/*
+ * Definition of the common QAT FW request parameter field.
+ *
+ * Service specific section of the request used across all of the services
+ * exposed by the QAT FW. Each of the services populates this block
+ * uniquely. Refer to service-specific header structures e.g.
+ * 'fw_comn_req_cipher_rqpars_s' (for Cipher) etc.
+ *
+ */
+struct fw_comn_req_rqpars
+{
+	/* LWs 14-26 */
+	uint32_t serv_specif_rqpars_lw[FW_NUM_LONGWORDS_13];
+
+};
+
+/*
+ * Definition of the common request structure with service specific
+ * fields
+ * This is a definition of the full qat request structure used by all
+ * services. Each service is free to use the service fields in its own
+ * way. This struct is useful as a message passing argument before the
+ * service contained within the request is determined.
+ */
+struct fw_comn_req
+{
+	/* LWs 0-1 */
+	struct fw_comn_req_hdr comn_hdr;
+	/* Common request header */
+
+	/* LWs 2-5 */
+	union fw_comn_req_hdr_cd_pars cd_pars;
+	/* Common Request content descriptor field which points either to a
+	 * content descriptor
+	 * parameter block or contains the service-specific data itself. */
+
+	/* LWs 6-13 */
+	struct fw_comn_req_mid comn_mid;
+	/* Common request middle section */
+
+	/* LWs 14-26 */
+	struct fw_comn_req_rqpars serv_specif_rqpars;
+	/* Common request service-specific parameter field */
+
+	/* LWs 27-31 */
+	struct fw_comn_req_cd_ctrl cd_ctrl;
+	/* Common request content descriptor control block -
+	 * this field is service-specific */
+
+};
+
+/*
+ * Error code field
+ *
+ * Overloaded field with 8 bit common error field or two
+ * 8 bit compression error fields for compression and translator slices
+ */
+union fw_comn_error {
+	struct
+	{
+		uint8_t resrvd;
+		/* 8 bit reserved field */
+
+		uint8_t comn_err_code;
+		/* 8 bit common error code */
+
+	} s;
+	/* Structure which is used for non-compression responses */
+
+	struct
+	{
+		uint8_t xlat_err_code;
+		/* 8 bit translator error field */
+
+		uint8_t cmp_err_code;
+		/* 8 bit compression error field */
+
+	} s1;
+	/* Structure which is used for compression responses */
+
+};
+
+/*
+ * Definition of the common QAT FW response header.
+ * This section of the response is common across all of the services
+ * that generate a firmware interface response
+ */
+struct fw_comn_resp_hdr
+{
+	/* LW0 */
+	uint8_t resrvd1;
+	/* Reserved field - this field is service-specific -
+	 * Note: The Response Destination Id has been removed
+	 * from first QWord */
+
+	uint8_t service_id;
+	/* Service Id returned by service block */
+
+	uint8_t response_type;
+	/* Response type - copied from the request to
+	 * the response message */
+
+	uint8_t hdr_flags;
+	/* This represents a flags field for the Response.
+	 * Bit<7> = 'valid' flag
+	 * Bit<6> = 'CNV' flag indicating that CNV was executed
+	 *          on the current request
+	 * Bit<5> = 'CNVNR' flag indicating that a recovery happened
+	 *          on the current request following a CNV error
+	 * All remaining bits are unused and are therefore reserved.
+	 * They must to be set to 0.
+	 */
+
+	/* LW 1 */
+	union fw_comn_error comn_error;
+	/* This field is overloaded to allow for one 8 bit common error field
+	 *   or two 8 bit error fields from compression and translator  */
+
+	uint8_t comn_status;
+	/* Status field which specifies which slice(s) report an error */
+
+	uint8_t cmd_id;
+	/* Command Id - passed from the request to the response message */
+
+};
+
+/*
+ * Definition of the common response structure with service specific
+ * fields
+ * This is a definition of the full qat response structure used by all
+ * services.
+ */
+struct fw_comn_resp
+{
+	/* LWs 0-1 */
+	struct fw_comn_resp_hdr comn_hdr;
+	/* Common header fields */
+
+	/* LWs 2-3 */
+	uint64_t opaque_data;
+	/* Opaque data passed from the request to the response message */
+
+	/* LWs 4-7 */
+	uint32_t resrvd[FW_NUM_LONGWORDS_4];
+	/* Reserved */
+
+};
+
+/*  Common QAT FW request header - structure of LW0
+ *  + ===== + ---- + ----------- + ----------- + ----------- + ----------- +
+ *  |  Bit  |  31  |  30 - 24    |  21 - 16    |  15 - 8     |  7 - 0      |
+ *  + ===== + ---- + ----------- + ----------- + ----------- + ----------- +
+ *  | Flags |  V   |   Reserved  | Serv Type   | Serv Cmd Id |  Reserved   |
+ *  + ===== + ---- + ----------- + ----------- + ----------- + ----------- +
+ */
+
+#define FW_COMN_VALID		__BIT(7)
+
+/*  Common QAT FW response header - structure of LW0
+ *  + ===== + --- + --- + ----- + ----- + --------- + ----------- + ----- +
+ *  |  Bit  | 31  | 30  |   29  | 28-24 |  21 - 16  |  15 - 8     |  7-0  |
+ *  + ===== + --- + ----+ ----- + ----- + --------- + ----------- + ----- +
+ *  | Flags |  V  | CNV | CNVNR | Rsvd  | Serv Type | Serv Cmd Id |  Rsvd |
+ *  + ===== + --- + --- + ----- + ----- + --------- + ----------- + ----- +  */
+/* Macros defining the bit position and mask of 'CNV' flag
+ * within the hdr_flags field of LW0 (service response only)  */
+#define FW_COMN_CNV_FLAG_BITPOS 6
+#define FW_COMN_CNV_FLAG_MASK 0x1
+
+/* Macros defining the bit position and mask of CNVNR flag
+ * within the hdr_flags field of LW0 (service response only)  */
+#define FW_COMN_CNVNR_FLAG_BITPOS 5
+#define FW_COMN_CNVNR_FLAG_MASK 0x1
+
+/*
+ * Macro for extraction of Service Type Field
+ *
+ * struct fw_comn_req_hdr  Structure 'fw_comn_req_hdr_t'
+ *                 to extract the Service Type Field
+ */
+#define FW_COMN_OV_SRV_TYPE_GET(fw_comn_req_hdr_t)	\
+	fw_comn_req_hdr_t.service_type
+
+/*
+ * Macro for setting of Service Type Field
+ *
+ * 'fw_comn_req_hdr_t' structure to set the Service
+ *                  Type Field
+ * val    Value of the Service Type Field
+ */
+#define FW_COMN_OV_SRV_TYPE_SET(fw_comn_req_hdr_t, val)	\
+	fw_comn_req_hdr_t.service_type = val
+
+/*
+ * Macro for extraction of Service Command Id Field
+ *
+ * struct fw_comn_req_hdr  Structure 'fw_comn_req_hdr_t'
+ *                 to extract the Service Command Id Field
+ */
+#define FW_COMN_OV_SRV_CMD_ID_GET(fw_comn_req_hdr_t)	\
+	fw_comn_req_hdr_t.service_cmd_id
+
+/*
+ * Macro for setting of Service Command Id Field
+ *
+ * 'fw_comn_req_hdr_t' structure to set the
+ *                  Service Command Id Field
+ * val    Value of the Service Command Id Field
+ */
+#define FW_COMN_OV_SRV_CMD_ID_SET(fw_comn_req_hdr_t, val)	\
+	fw_comn_req_hdr_t.service_cmd_id = val
+
+/*
+ * Extract the valid flag from the request or response's header flags.
+ *
+ * hdr_t  Request or Response 'hdr_t' structure to extract the valid bit
+ *  from the  'hdr_flags' field.
+ */
+#define FW_COMN_HDR_VALID_FLAG_GET(hdr_t)	\
+	FW_COMN_VALID_FLAG_GET(hdr_t.hdr_flags)
+
+/*
+ * Extract the CNVNR flag from the header flags in the response only.
+ *
+ * hdr_t  Response 'hdr_t' structure to extract the CNVNR bit
+ *  from the  'hdr_flags' field.
+ */
+#define FW_COMN_HDR_CNVNR_FLAG_GET(hdr_flags)	\
+	FIELD_GET(hdr_flags,	\
+			      FW_COMN_CNVNR_FLAG_BITPOS,	\
+			      FW_COMN_CNVNR_FLAG_MASK)
+
+/*
+ * Extract the CNV flag from the header flags in the response only.
+ *
+ * hdr_t  Response 'hdr_t' structure to extract the CNV bit
+ *  from the  'hdr_flags' field.
+ */
+#define FW_COMN_HDR_CNV_FLAG_GET(hdr_flags)	\
+	FIELD_GET(hdr_flags,	\
+			      FW_COMN_CNV_FLAG_BITPOS,	\
+			      FW_COMN_CNV_FLAG_MASK)
+
+/*
+ * Set the valid bit in the request's header flags.
+ *
+ * hdr_t  Request or Response 'hdr_t' structure to set the valid bit
+ * val    Value of the valid bit flag.
+ */
+#define FW_COMN_HDR_VALID_FLAG_SET(hdr_t, val)	\
+	FW_COMN_VALID_FLAG_SET(hdr_t, val)
+
+/*
+ * Common macro to extract the valid flag from the header flags field
+ * within the header structure (request or response).
+ *
+ * hdr_t  Structure (request or response) to extract the
+ *  valid bit from the 'hdr_flags' field.
+ */
+#define FW_COMN_VALID_FLAG_GET(hdr_flags)	\
+	FIELD_GET(hdr_flags,	\
+			      FW_COMN_VALID_FLAG_BITPOS,	\
+			      FW_COMN_VALID_FLAG_MASK)
+
+/*
+ * Common macro to extract the remaining reserved flags from the header
+ * flags field within the header structure (request or response).
+ *
+ * hdr_t  Structure (request or response) to extract the
+ *  remaining bits from the 'hdr_flags' field (excluding the
+ *  valid flag).
+ */
+#define FW_COMN_HDR_RESRVD_FLD_GET(hdr_flags)	\
+	(hdr_flags & FW_COMN_HDR_RESRVD_FLD_MASK)
+
+/*
+ * Common macro to set the valid bit in the header flags field within
+ * the header structure (request or response).
+ *
+ * hdr_t  Structure (request or response) containing the header
+ *  flags field, to allow the valid bit to be set.
+ * val    Value of the valid bit flag.
+ */
+#define FW_COMN_VALID_FLAG_SET(hdr_t, val)	\
+	FIELD_SET((hdr_t.hdr_flags),	\
+			      (val),	\
+			      FW_COMN_VALID_FLAG_BITPOS,	\
+			      FW_COMN_VALID_FLAG_MASK)
+
+/*
+ * Macro that must be used when building the common header flags.
+ * Note that all bits reserved field bits 0-6 (LW0) need to be forced to 0.
+ *
+ * ptr   Value of the valid flag
+ */
+
+#define FW_COMN_HDR_FLAGS_BUILD(valid)	\
+	(((valid)&FW_COMN_VALID_FLAG_MASK)	\
+	 << FW_COMN_VALID_FLAG_BITPOS)
+
+/*
+ *  Common Request Flags Definition
+ *  The bit offsets below are within the flags field. These are NOT relative to
+ *  the memory word. Unused fields e.g. reserved bits, must be zeroed.
+ *
+ *  + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- +
+ *  | Bits [15:8]    |  15 |  14 |  13 |  12 |  11 |  10 |  9  |  8  |
+ *  + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- +
+ *  | Flags[15:8]    | Rsv | Rsv | Rsv | Rsv | Rsv | Rsv | Rsv | Rsv |
+ *  + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- +
+ *  | Bits  [7:0]    |  7  |  6  |  5  |  4  |  3  |  2  |  1  |  0  |
+ *  + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- +
+ *  | Flags [7:0]    | Rsv | Rsv | Rsv | Rsv | Rsv | BnP | Cdt | Ptr |
+ *  + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- +
+ */
+
+#define COMN_PTR_TYPE_BITPOS 0
+/* Common Request Flags - Starting bit position indicating
+ * Src&Dst Buffer Pointer type   */
+
+#define COMN_PTR_TYPE_MASK 0x1
+/* Common Request Flags - One bit mask used to determine
+ * Src&Dst Buffer Pointer type  */
+
+#define COMN_CD_FLD_TYPE_BITPOS 1
+/* Common Request Flags - Starting bit position indicating
+ * CD Field type   */
+
+#define COMN_CD_FLD_TYPE_MASK 0x1
+/* Common Request Flags - One bit mask used to determine
+ * CD Field type  */
+
+#define COMN_BNP_ENABLED_BITPOS 2
+/* Common Request Flags - Starting bit position indicating
+ * the source buffer contains batch of requests. if this
+ * bit is set, source buffer is type of Batch And Pack OpData List
+ * and the Ptr Type Bit only applies to Destination buffer.  */
+
+#define COMN_BNP_ENABLED_MASK 0x1
+/* Batch And Pack Enabled Flag Mask - One bit mask used to determine
+ * the source buffer is in Batch and Pack OpData Link List Mode.  */
+
+/* ========================================================================= */
+/*                                       Pointer Type Flag definitions       */
+/* ========================================================================= */
+#define COMN_PTR_TYPE_FLAT 0x0
+/* Constant value indicating Src&Dst Buffer Pointer type is flat
+ * If Batch and Pack mode is enabled, only applies to Destination buffer. */
+
+#define COMN_PTR_TYPE_SGL 0x1
+/* Constant value indicating Src&Dst Buffer Pointer type is SGL type
+ * If Batch and Pack mode is enabled, only applies to Destination buffer. */
+
+#define COMN_PTR_TYPE_BATCH 0x2
+/* Constant value indicating Src is a batch request
+ * and Dst Buffer Pointer type is SGL type  */
+
+/* ========================================================================= */
+/*                                       CD Field Flag definitions           */
+/* ========================================================================= */
+#define COMN_CD_FLD_TYPE_64BIT_ADR 0x0
+/* Constant value indicating CD Field contains 64-bit address  */
+
+#define COMN_CD_FLD_TYPE_16BYTE_DATA 0x1
+/* Constant value indicating CD Field contains 16 bytes of setup data  */
+
+/* ========================================================================= */
+/*                       Batch And Pack Enable/Disable Definitions           */
+/* ========================================================================= */
+#define COMN_BNP_ENABLED 0x1
+/* Constant value indicating Source buffer will point to Batch And Pack OpData
+ * List  */
+
+#define COMN_BNP_DISABLED 0x0
+/* Constant value indicating Source buffer will point to Batch And Pack OpData
+ * List  */
+
+/*
+ * Macro that must be used when building the common request flags (for all
+ * requests but comp BnP).
+ * Note that all bits reserved field bits 2-15 (LW1) need to be forced to 0.
+ *
+ * ptr   Value of the pointer type flag
+ * cdt   Value of the cd field type flag
+*/
+#define FW_COMN_FLAGS_BUILD(cdt, ptr)	\
+	((((cdt)&COMN_CD_FLD_TYPE_MASK) << COMN_CD_FLD_TYPE_BITPOS) |	\
+	 (((ptr)&COMN_PTR_TYPE_MASK) << COMN_PTR_TYPE_BITPOS))
+
+/*
+ * Macro that must be used when building the common request flags for comp
+ * BnP service.
+ * Note that all bits reserved field bits 3-15 (LW1) need to be forced to 0.
+ *
+ * ptr   Value of the pointer type flag
+ * cdt   Value of the cd field type flag
+ * bnp   Value of the bnp enabled flag
+ */
+#define FW_COMN_FLAGS_BUILD_BNP(cdt, ptr, bnp)	\
+	((((cdt)&COMN_CD_FLD_TYPE_MASK) << COMN_CD_FLD_TYPE_BITPOS) |	\
+	 (((ptr)&COMN_PTR_TYPE_MASK) << COMN_PTR_TYPE_BITPOS) |	\
+	 (((bnp)&COMN_BNP_ENABLED_MASK) << COMN_BNP_ENABLED_BITPOS))
+
+/*
+ * Macro for extraction of the pointer type bit from the common flags
+ *
+ * flags      Flags to extract the pointer type bit from
+ */
+#define FW_COMN_PTR_TYPE_GET(flags)	\
+	FIELD_GET(flags, COMN_PTR_TYPE_BITPOS, COMN_PTR_TYPE_MASK)
+
+/*
+ * Macro for extraction of the cd field type bit from the common flags
+ *
+ * flags      Flags to extract the cd field type type bit from
+ */
+#define FW_COMN_CD_FLD_TYPE_GET(flags)	\
+	FIELD_GET(flags, COMN_CD_FLD_TYPE_BITPOS, COMN_CD_FLD_TYPE_MASK)
+
+/*
+ * Macro for extraction of the bnp field type bit from the common flags
+ *
+ * flags      Flags to extract the bnp field type type bit from
+ *
+ */
+#define FW_COMN_BNP_ENABLED_GET(flags)	\
+	FIELD_GET(flags, COMN_BNP_ENABLED_BITPOS, COMN_BNP_ENABLED_MASK)
+
+/*
+ * Macro for setting the pointer type bit in the common flags
+ *
+ * flags      Flags in which Pointer Type bit will be set
+ * val        Value of the bit to be set in flags
+ *
+ */
+#define FW_COMN_PTR_TYPE_SET(flags, val)	\
+	FIELD_SET(flags, val, COMN_PTR_TYPE_BITPOS, COMN_PTR_TYPE_MASK)
+
+/*
+ * Macro for setting the cd field type bit in the common flags
+ *
+ * flags      Flags in which Cd Field Type bit will be set
+ * val        Value of the bit to be set in flags
+ *
+ */
+#define FW_COMN_CD_FLD_TYPE_SET(flags, val)	\
+	FIELD_SET(	\
+		flags, val, COMN_CD_FLD_TYPE_BITPOS, COMN_CD_FLD_TYPE_MASK)
+
+/*
+ * Macro for setting the bnp field type bit in the common flags
+ *
+ * flags      Flags in which Bnp Field Type bit will be set
+ * val        Value of the bit to be set in flags
+ *
+ */
+#define FW_COMN_BNP_ENABLE_SET(flags, val)	\
+	FIELD_SET(	\
+		flags, val, COMN_BNP_ENABLED_BITPOS, COMN_BNP_ENABLED_MASK)
+
+/*
+ * Macros using the bit position and mask to set/extract the next
+ * and current id nibbles within the next_curr_id field of the
+ * content descriptor header block. Note that these are defined
+ * in the common header file, as they are used by compression, cipher
+ * and authentication.
+ *
+ * cd_ctrl_hdr_t      Content descriptor control block header pointer.
+ * val                Value of the field being set.
+ */
+#define FW_COMN_NEXT_ID_BITPOS 4
+#define FW_COMN_NEXT_ID_MASK 0xF0
+#define FW_COMN_CURR_ID_BITPOS 0
+#define FW_COMN_CURR_ID_MASK 0x0F
+
+#define FW_COMN_NEXT_ID_GET(cd_ctrl_hdr_t)	\
+	((((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_NEXT_ID_MASK) >>	\
+	 (FW_COMN_NEXT_ID_BITPOS))
+
+#define FW_COMN_NEXT_ID_SET(cd_ctrl_hdr_t, val)	\
+	((cd_ctrl_hdr_t)->next_curr_id) =	\
+		((((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_CURR_ID_MASK) |	\
+		 ((val << FW_COMN_NEXT_ID_BITPOS) &	\
+		  FW_COMN_NEXT_ID_MASK))
+
+#define FW_COMN_CURR_ID_GET(cd_ctrl_hdr_t)	\
+	(((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_CURR_ID_MASK)
+
+#define FW_COMN_CURR_ID_SET(cd_ctrl_hdr_t, val)	\
+	((cd_ctrl_hdr_t)->next_curr_id) =	\
+		((((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_NEXT_ID_MASK) |	\
+		 ((val)&FW_COMN_CURR_ID_MASK))
+
+/*
+ *  Common Status Field Definition  The bit offsets below are within the COMMON
+ *  RESPONSE status field, assumed to be 8 bits wide. In the case of the PKE
+ *  response (which follows the CPM 1.5 message format), the status field is 16
+ *  bits wide.
+ *  The status flags are contained within the most significant byte and align
+ *  with the diagram below. Please therefore refer to the service-specific PKE
+ *  header file for the appropriate macro definition to extract the PKE status
+ *  flag from the PKE response, which assumes that a word is passed to the
+ *  macro.
+ *  + ===== + ------ + --- + --- + ---- + ---- + -------- + ---- + ---------- +
+ *  |  Bit  |   7    |  6  |  5  |  4   |  3   |    2     |   1  |      0     |
+ *  + ===== + ------ + --- + --- + ---- + ---- + -------- + ---- + ---------- +
+ *  | Flags | Crypto | Pke | Cmp | Xlat | EOLB | UnSupReq | Rsvd | XltWaApply |
+ *  + ===== + ------ + --- + --- + ---- + ---- + -------- + ---- + ---------- +
+ * Note:
+ * For the service specific status bit definitions refer to service header files
+ * Eg. Crypto Status bit refers to Symmetric Crypto, Key Generation, and NRBG
+ * Requests' Status. Unused bits e.g. reserved bits need to have been forced to
+ * 0.
+ */
+
+#define COMN_RESP_CRYPTO_STATUS_BITPOS 7
+/* Starting bit position indicating Response for Crypto service Flag  */
+
+#define COMN_RESP_CRYPTO_STATUS_MASK 0x1
+/* One bit mask used to determine Crypto status mask  */
+
+#define COMN_RESP_PKE_STATUS_BITPOS 6
+/* Starting bit position indicating Response for PKE service Flag  */
+
+#define COMN_RESP_PKE_STATUS_MASK 0x1
+/* One bit mask used to determine PKE status mask  */
+
+#define COMN_RESP_CMP_STATUS_BITPOS 5
+/* Starting bit position indicating Response for Compression service Flag  */
+
+#define COMN_RESP_CMP_STATUS_MASK 0x1
+/* One bit mask used to determine Compression status mask  */
+
+#define COMN_RESP_XLAT_STATUS_BITPOS 4
+/* Starting bit position indicating Response for Xlat service Flag  */
+
+#define COMN_RESP_XLAT_STATUS_MASK 0x1
+/* One bit mask used to determine Translator status mask  */
+
+#define COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS 3
+/* Starting bit position indicating the last block in a deflate stream for
+  the compression service Flag */
+
+#define COMN_RESP_CMP_END_OF_LAST_BLK_MASK 0x1
+/* One bit mask used to determine the last block in a deflate stream
+   status mask */
+
+#define COMN_RESP_UNSUPPORTED_REQUEST_BITPOS 2
+/* Starting bit position indicating when an unsupported service request Flag  */
+
+#define COMN_RESP_UNSUPPORTED_REQUEST_MASK 0x1
+/* One bit mask used to determine the unsupported service request status mask  */
+
+#define COMN_RESP_XLT_WA_APPLIED_BITPOS 0
+/* Bit position indicating a firmware workaround was applied to translation  */
+
+#define COMN_RESP_XLT_WA_APPLIED_MASK 0x1
+/* One bit mask  */
+
+/*
+ * Macro that must be used when building the status
+ * for the common response
+ *
+ * crypto   Value of the Crypto Service status flag
+ * comp     Value of the Compression Service Status flag
+ * xlat     Value of the Xlator Status flag
+ * eolb     Value of the Compression End of Last Block Status flag
+ * unsupp   Value of the Unsupported Request flag
+ * xlt_wa   Value of the Translation WA marker
+ */
+#define FW_COMN_RESP_STATUS_BUILD(	\
+	crypto, pke, comp, xlat, eolb, unsupp, xlt_wa)	\
+	((((crypto)&COMN_RESP_CRYPTO_STATUS_MASK)	\
+	  << COMN_RESP_CRYPTO_STATUS_BITPOS) |	\
+	 (((pke)&COMN_RESP_PKE_STATUS_MASK)	\
+	  << COMN_RESP_PKE_STATUS_BITPOS) |	\
+	 (((xlt_wa)&COMN_RESP_XLT_WA_APPLIED_MASK)	\
+	  << COMN_RESP_XLT_WA_APPLIED_BITPOS) |	\
+	 (((comp)&COMN_RESP_CMP_STATUS_MASK)	\
+	  << COMN_RESP_CMP_STATUS_BITPOS) |	\
+	 (((xlat)&COMN_RESP_XLAT_STATUS_MASK)	\
+	  << COMN_RESP_XLAT_STATUS_BITPOS) |	\
+	 (((eolb)&COMN_RESP_CMP_END_OF_LAST_BLK_MASK)	\
+	  << COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS) |	\
+	 (((unsupp)&COMN_RESP_UNSUPPORTED_REQUEST_BITPOS)	\
+	  << COMN_RESP_UNSUPPORTED_REQUEST_MASK))
+
+/*
+ * Macro for extraction of the Crypto bit from the status
+ *
+ * status Status to extract the status bit from
+ */
+#define FW_COMN_RESP_CRYPTO_STAT_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_CRYPTO_STATUS_BITPOS,	\
+			      COMN_RESP_CRYPTO_STATUS_MASK)
+
+/*
+ * Macro for extraction of the PKE bit from the status
+ *
+ * status Status to extract the status bit from
+ */
+#define FW_COMN_RESP_PKE_STAT_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_PKE_STATUS_BITPOS,	\
+			      COMN_RESP_PKE_STATUS_MASK)
+
+/*
+ * Macro for extraction of the Compression bit from the status
+ *
+ * status Status to extract the status bit from
+ */
+#define FW_COMN_RESP_CMP_STAT_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_CMP_STATUS_BITPOS,	\
+			      COMN_RESP_CMP_STATUS_MASK)
+
+/*
+ * Macro for extraction of the Translator bit from the status
+ *
+ * status Status to extract the status bit from
+ */
+#define FW_COMN_RESP_XLAT_STAT_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_XLAT_STATUS_BITPOS,	\
+			      COMN_RESP_XLAT_STATUS_MASK)
+
+/*
+ * Macro for extraction of the Translation Workaround Applied bit from the
+ * status
+ *
+ * status Status to extract the status bit from
+ */
+#define FW_COMN_RESP_XLT_WA_APPLIED_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_XLT_WA_APPLIED_BITPOS,	\
+			      COMN_RESP_XLT_WA_APPLIED_MASK)
+
+/*
+ * Macro for extraction of the end of compression block bit from the
+ * status
+ *
+ * status
+ * Status to extract the status bit from
+ */
+#define FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS,	\
+			      COMN_RESP_CMP_END_OF_LAST_BLK_MASK)
+
+/*
+ * Macro for extraction of the Unsupported request from the status
+ *
+ * status
+ * Status to extract the status bit from
+ */
+#define FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(status)	\
+	FIELD_GET(status,	\
+			      COMN_RESP_UNSUPPORTED_REQUEST_BITPOS,	\
+			      COMN_RESP_UNSUPPORTED_REQUEST_MASK)
+
+#define FW_COMN_STATUS_FLAG_OK 0
+/* Definition of successful processing of a request  */
+
+#define FW_COMN_STATUS_FLAG_ERROR 1
+/* Definition of erroneous processing of a request  */
+
+#define FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_CLR 0
+/* Final Deflate block of a compression request not completed  */
+
+#define FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET 1
+/* Final Deflate block of a compression request completed  */
+
+#define ERR_CODE_NO_ERROR 0
+/* Error Code constant value for no error  */
+
+#define ERR_CODE_INVALID_BLOCK_TYPE -1
+/* Invalid block type (type == 3)*/
+
+#define ERR_CODE_NO_MATCH_ONES_COMP -2
+/* Stored block length does not match one's complement */
+
+#define ERR_CODE_TOO_MANY_LEN_OR_DIS -3
+/* Too many length or distance codes */
+
+#define ERR_CODE_INCOMPLETE_LEN -4
+/* Code lengths codes incomplete */
+
+#define ERR_CODE_RPT_LEN_NO_FIRST_LEN -5
+/* Repeat lengths with no first length */
+
+#define ERR_CODE_RPT_GT_SPEC_LEN -6
+/* Repeat more than specified lengths */
+
+#define ERR_CODE_INV_LIT_LEN_CODE_LEN -7
+/* Invalid lit/len code lengths */
+
+#define ERR_CODE_INV_DIS_CODE_LEN -8
+/* Invalid distance code lengths */
+
+#define ERR_CODE_INV_LIT_LEN_DIS_IN_BLK -9
+/* Invalid lit/len or distance code in fixed/dynamic block */
+
+#define ERR_CODE_DIS_TOO_FAR_BACK -10
+/* Distance too far back in fixed or dynamic block */
+
+/* Common Error code definitions */
+#define ERR_CODE_OVERFLOW_ERROR -11
+/* Error Code constant value for overflow error  */
+
+#define ERR_CODE_SOFT_ERROR -12
+/* Error Code constant value for soft error  */
+
+#define ERR_CODE_FATAL_ERROR -13
+/* Error Code constant value for hard/fatal error  */
+
+#define ERR_CODE_COMP_OUTPUT_CORRUPTION -14
+/* Error Code constant for compression output corruption */
+
+#define ERR_CODE_HW_INCOMPLETE_FILE -15
+/* Error Code constant value for incomplete file hardware error  */
+
+#define ERR_CODE_SSM_ERROR -16
+/* Error Code constant value for error detected by SSM e.g. slice hang  */
+
+#define ERR_CODE_ENDPOINT_ERROR -17
+/* Error Code constant value for error detected by PCIe Endpoint, e.g. push
+ * data error   */
+
+#define ERR_CODE_CNV_ERROR -18
+/* Error Code constant value for cnv failure  */
+
+#define ERR_CODE_EMPTY_DYM_BLOCK -19
+/* Error Code constant value for submission of empty dynamic stored block to
+ * slice   */
+
+#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_HANDLE -20
+/* Error Code constant for invalid handle in kpt crypto service */
+
+#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_HMAC_FAILED -21
+/* Error Code constant for failed hmac in kpt crypto service */
+
+#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_WRAPPING_ALGO -22
+/* Error Code constant for invalid wrapping algo in kpt crypto service */
+
+#define ERR_CODE_KPT_DRNG_SEED_NOT_LOAD -23
+/* Error Code constant for no drng seed is not loaded in kpt ecdsa signrs
+/service */
+
+#define FW_LA_ICV_VER_STATUS_PASS FW_COMN_STATUS_FLAG_OK
+/* Status flag indicating that the ICV verification passed  */
+
+#define FW_LA_ICV_VER_STATUS_FAIL FW_COMN_STATUS_FLAG_ERROR
+/* Status flag indicating that the ICV verification failed  */
+
+#define FW_LA_TRNG_STATUS_PASS FW_COMN_STATUS_FLAG_OK
+/* Status flag indicating that the TRNG returned valid entropy data  */
+
+#define FW_LA_TRNG_STATUS_FAIL FW_COMN_STATUS_FLAG_ERROR
+/* Status flag indicating that the TRNG Command Failed.  */
+
+/* -------------------------------------------------------------------------- */
+
+/*
+ * Definition of the full bulk processing request structure.
+ * Used for hash, cipher, hash-cipher and authentication-encryption
+ * requests etc.
+ */
+struct fw_la_bulk_req
+{
+	/* LWs 0-1 */
+	struct fw_comn_req_hdr comn_hdr;
+	/* Common request header - for Service Command Id,
+	 * use service-specific Crypto Command Id.
+	 * Service Specific Flags - use Symmetric Crypto Command Flags
+	 * (all of cipher, auth, SSL3, TLS and MGF,
+	 * excluding TRNG - field unused) */
+
+	/* LWs 2-5 */
+	union fw_comn_req_hdr_cd_pars cd_pars;
+	/* Common Request content descriptor field which points either to a
+	 * content descriptor
+	 * parameter block or contains the service-specific data itself. */
+
+	/* LWs 6-13 */
+	struct fw_comn_req_mid comn_mid;
+	/* Common request middle section */
+
+	/* LWs 14-26 */
+	struct fw_comn_req_rqpars serv_specif_rqpars;
+	/* Common request service-specific parameter field */
+
+	/* LWs 27-31 */
+	struct fw_comn_req_cd_ctrl cd_ctrl;
+	/* Common request content descriptor control block -
+	 * this field is service-specific */
+
+};
+
+/* clang-format off */
+
+/*
+ *  LA BULK (SYMMETRIC CRYPTO) COMMAND FLAGS
+ *
+ *  + ===== + ---------- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- +
+ *  |  Bit  |   [15:13]  |  12   |  11   |  10   |  7-9  |   6   |   5   |   4   |  3    |   2   |  1-0  |
+ *  + ===== + ---------- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ------+ ----- +
+ *  | Flags | Resvd Bits | ZUC   | GcmIV |Digest | Prot  | Cmp   | Rtn   | Upd   | Ciph/ | CiphIV| Part- |
+ *  |       |     =0     | Prot  | Len   | In Buf| flgs  | Auth  | Auth  | State | Auth  | Field |  ial  |
+ *  + ===== + ---------- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ------+ ----- +
+ */
+
+/* clang-format on */
+
+/* Private defines */
+
+#define FW_LA_ZUC_3G_PROTO			__BIT(12)
+/* Indicating ZUC processing for a encrypt command
+ * Must be set for Cipher-only, Cipher + Auth and Auth-only   */
+
+#define FW_LA_GCM_IV_LEN_12_OCTETS		__BIT(11)
+/* Indicates the IV Length for GCM protocol is 96 Bits (12 Octets)
+ * If set FW does the padding to compute CTR0  */
+
+#define FW_LA_DIGEST_IN_BUFFER			__BIT(10)
+/* Flag representing that authentication digest is stored or is extracted
+ * from the source buffer. Auth Result Pointer will be ignored in this case.  */
+
+#define FW_LA_PROTO				__BITS(7, 9)
+#define FW_LA_PROTO_SNOW_3G			__BIT(9)
+/* Indicates SNOW_3G processing for a encrypt command  */
+#define FW_LA_PROTO_GCM				__BIT(8)
+/* Indicates GCM processing for a auth_encrypt command  */
+#define FW_LA_PROTO_CCM				__BIT(7)
+/* Indicates CCM processing for a auth_encrypt command  */
+#define FW_LA_PROTO_NONE			0
+/* Indicates no specific protocol processing for the command  */
+
+#define FW_LA_CMP_AUTH_RES			__BIT(6)
+/* Flag representing the need to compare the auth result data to the expected
+ * value in DRAM at the auth_address.  */
+
+#define FW_LA_RET_AUTH_RES			__BIT(5)
+/* Flag representing the need to return the auth result data to dram after the
+ * request processing is complete  */
+
+#define FW_LA_UPDATE_STATE			__BIT(4)
+/* Flag representing the need to update the state data in dram after the
+ * request processing is complete  */
+
+#define FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP	__BIT(3)
+/* Flag representing Cipher/Auth Config Offset Type, where the offset
+ * is contained in SHRAM constants page. When the SHRAM constants page
+ * is not used for cipher/auth configuration, then the Content Descriptor
+ * pointer field must be a pointer (as opposed to a 16-byte key), since
+ * the block pointed to must contain both the slice config and the key  */
+
+#define FW_CIPH_IV_16BYTE_DATA			__BIT(2)
+/* Flag representing Cipher IV field contents as 16-byte data array
+ * Otherwise Cipher IV field contents via 64-bit pointer */
+
+#define FW_LA_PARTIAL				__BITS(0, 1)
+#define FW_LA_PARTIAL_NONE			0
+/* Flag representing no need for partial processing condition i.e.
+ * entire packet processed in the current command  */
+#define FW_LA_PARTIAL_START			1
+/* Flag representing the first chunk of the partial packet  */
+#define FW_LA_PARTIAL_MID			3
+/* Flag representing a middle chunk of the partial packet  */
+#define FW_LA_PARTIAL_END			2
+/* Flag representing the final/end chunk of the partial packet  */
+
+/* The table below defines the meaning of the prefix_addr & hash_state_sz in
+ * the case of partial processing. See the HLD for further details
+ *
+ *  + ====== + ------------------------- + ----------------------- +
+ *  | Parial |       Prefix Addr         |       Hash State Sz     |
+ *  | State  |                           |                         |
+ *  + ====== + ------------------------- + ----------------------- +
+ *  |  FULL  | Points to the prefix data | Prefix size as below.   |
+ *  |        |                           | No update of state      |
+ *  + ====== + ------------------------- + ----------------------- +
+ *  |  SOP   | Points to the prefix      | = inner prefix rounded  |
+ *  |        | data. State is updated    | to qwrds + outer prefix |
+ *  |        | at prefix_addr - state_sz | rounded to qwrds. The   |
+ *  |        | - 8 (counter size)        | writeback state sz      |
+ *  |        |                           | comes from the CD       |
+ *  + ====== + ------------------------- + ----------------------- +
+ *  |  MOP   | Points to the state data  | State size rounded to   |
+ *  |        | Updated state written to  | num qwrds + 8 (for the  |
+ *  |        | same location             | counter) + inner prefix |
+ *  |        |                           | rounded to qwrds +      |
+ *  |        |                           | outer prefix rounded to |
+ *  |        |                           | qwrds.                  |
+ *  + ====== + ------------------------- + ----------------------- +
+ *  |  EOP   | Points to the state data  | State size rounded to   |
+ *  |        |                           | num qwrds + 8 (for the  |
+ *  |        |                           | counter) + inner prefix |
+ *  |        |                           | rounded to qwrds +      |
+ *  |        |                           | outer prefix rounded to |
+ *  |        |                           | qwrds.                  |
+ *  + ====== + ------------------------- + ----------------------- +
+ *
+ *  Notes:
+ *
+ *  - If the EOP is set it is assumed that no state update is to be performed.
+ *    However it is the clients responsibility to set the update_state flag
+ *    correctly i.e. not set for EOP or Full packet cases. Only set for SOP and
+ *    MOP with no EOP flag
+ *  - The SOP take precedence over the MOP and EOP i.e. in the calculation of
+ *    the address to writeback the state.
+ *  - The prefix address must be on at least the 8 byte boundary
+ */
+
+/* Macros for extracting field bits */
+/*
+ *   Macro for extraction of the Cipher IV field contents (bit 2)
+ *
+ * flags        Flags to extract the Cipher IV field contents
+ *
+ */
+#define FW_LA_CIPH_IV_FLD_FLAG_GET(flags)	\
+	FIELD_GET(flags, LA_CIPH_IV_FLD_BITPOS, LA_CIPH_IV_FLD_MASK)
+
+/*
+ *   Macro for extraction of the Cipher/Auth Config
+ *   offset type (bit 3)
+ *
+ * flags        Flags to extract the Cipher/Auth Config offset type
+ *
+ */
+#define FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_GET(flags)	\
+	FIELD_GET(flags,	\
+	    LA_CIPH_AUTH_CFG_OFFSET_BITPOS,	\
+	    LA_CIPH_AUTH_CFG_OFFSET_MASK)
+
+/*
+ * Macro for extraction of the ZUC protocol bit
+ * information (bit 11)
+ *
+ * flags        Flags to extract the ZUC protocol bit
+ */
+#define FW_LA_ZUC_3G_PROTO_FLAG_GET(flags)	\
+	FIELD_GET(flags,	\
+	    FW_LA_ZUC_3G_PROTO_FLAG_BITPOS,	\
+	    FW_LA_ZUC_3G_PROTO_FLAG_MASK)
+
+/*
+ * Macro for extraction of the GCM IV Len is 12 Octets / 96 Bits
+ * information (bit 11)
+ *
+ * flags        Flags to extract the GCM IV length
+ */
+#define FW_LA_GCM_IV_LEN_FLAG_GET(flags)	\
+	FIELD_GET(	\
+		flags, LA_GCM_IV_LEN_FLAG_BITPOS, LA_GCM_IV_LEN_FLAG_MASK)
+
+/*
+ * Macro for extraction of the LA protocol state (bits 9-7)
+ *
+ * flags        Flags to extract the protocol state
+ */
+#define FW_LA_PROTO_GET(flags)	\
+	FIELD_GET(flags, LA_PROTO_BITPOS, LA_PROTO_MASK)
+
+/*
+ * Macro for extraction of the "compare auth" state (bit 6)
+ *
+ * flags        Flags to extract the compare auth result state
+ *
+ */
+#define FW_LA_CMP_AUTH_GET(flags)	\
+	FIELD_GET(flags, LA_CMP_AUTH_RES_BITPOS, LA_CMP_AUTH_RES_MASK)
+
+/*
+ * Macro for extraction of the "return auth" state (bit 5)
+ *
+ * flags        Flags to extract the return auth result state
+ *
+ */
+#define FW_LA_RET_AUTH_GET(flags)	\
+	FIELD_GET(flags, LA_RET_AUTH_RES_BITPOS, LA_RET_AUTH_RES_MASK)
+
+/*
+ * Macro for extraction of the "digest in buffer" state (bit 10)
+ *
+ * flags     Flags to extract the digest in buffer state
+ *
+ */
+#define FW_LA_DIGEST_IN_BUFFER_GET(flags)	\
+	FIELD_GET(	\
+		flags, LA_DIGEST_IN_BUFFER_BITPOS, LA_DIGEST_IN_BUFFER_MASK)
+
+/*
+ * Macro for extraction of the update content state value. (bit 4)
+ *
+ * flags        Flags to extract the update content state bit
+ */
+#define FW_LA_UPDATE_STATE_GET(flags)	\
+	FIELD_GET(flags, LA_UPDATE_STATE_BITPOS, LA_UPDATE_STATE_MASK)
+
+/*
+ * Macro for extraction of the "partial" packet state (bits 1-0)
+ *
+ * flags        Flags to extract the partial state
+ */
+#define FW_LA_PARTIAL_GET(flags)	\
+	FIELD_GET(flags, LA_PARTIAL_BITPOS, LA_PARTIAL_MASK)
+
+/* Macros for setting field bits */
+/*
+ *   Macro for setting the Cipher IV field contents
+ *
+ * flags        Flags to set with the Cipher IV field contents
+ * val          Field contents indicator value
+ */
+#define FW_LA_CIPH_IV_FLD_FLAG_SET(flags, val)	\
+	FIELD_SET(	\
+		flags, val, LA_CIPH_IV_FLD_BITPOS, LA_CIPH_IV_FLD_MASK)
+
+/*
+ * Macro for setting the Cipher/Auth Config
+ * offset type
+ *
+ * flags        Flags to set the Cipher/Auth Config offset type
+ * val          Offset type value
+ */
+#define FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(flags, val)	\
+	FIELD_SET(flags,	\
+	    val,	\
+	    LA_CIPH_AUTH_CFG_OFFSET_BITPOS,	\
+	    LA_CIPH_AUTH_CFG_OFFSET_MASK)
+
+/*
+ * Macro for setting the ZUC protocol flag
+ *
+ * flags      Flags to set the ZUC protocol flag
+ * val        Protocol value
+ */
+#define FW_LA_ZUC_3G_PROTO_FLAG_SET(flags, val)	\
+	FIELD_SET(flags,	\
+	    val,	\
+	    FW_LA_ZUC_3G_PROTO_FLAG_BITPOS,	\
+	    FW_LA_ZUC_3G_PROTO_FLAG_MASK)
+
+/*
+ * Macro for setting the GCM IV length flag state
+ *
+ * flags      Flags to set the GCM IV length flag state
+ * val        Protocol value
+ */
+#define FW_LA_GCM_IV_LEN_FLAG_SET(flags, val)	\
+	FIELD_SET(flags,	\
+	    val,	\
+	    LA_GCM_IV_LEN_FLAG_BITPOS,	\
+	    LA_GCM_IV_LEN_FLAG_MASK)
+
+/*
+ * Macro for setting the LA protocol flag state
+ *
+ * flags        Flags to set the protocol state
+ * val          Protocol value
+ */
+#define FW_LA_PROTO_SET(flags, val)	\
+	FIELD_SET(flags, val, LA_PROTO_BITPOS, LA_PROTO_MASK)
+
+/*
+ * Macro for setting the "compare auth" flag state
+ *
+ * flags      Flags to set the compare auth result state
+ * val        Compare Auth value
+ */
+#define FW_LA_CMP_AUTH_SET(flags, val)	\
+	FIELD_SET(	\
+		flags, val, LA_CMP_AUTH_RES_BITPOS, LA_CMP_AUTH_RES_MASK)
+
+/*
+ * Macro for setting the "return auth" flag state
+ *
+ * flags      Flags to set the return auth result state
+ * val        Return Auth value
+ */
+#define FW_LA_RET_AUTH_SET(flags, val)	\
+	FIELD_SET(	\
+		flags, val, LA_RET_AUTH_RES_BITPOS, LA_RET_AUTH_RES_MASK)
+
+/*
+ * Macro for setting the "digest in buffer" flag state
+ *
+ * flags     Flags to set the digest in buffer state
+ * val       Digest in buffer value
+ */
+#define FW_LA_DIGEST_IN_BUFFER_SET(flags, val)	\
+	FIELD_SET(flags,	\
+	    val,	\
+	    LA_DIGEST_IN_BUFFER_BITPOS,	\
+	    LA_DIGEST_IN_BUFFER_MASK)
+
+/*
+ *   Macro for setting the "update state" flag value
+ *
+ * flags      Flags to set the update content state
+ * val        Update Content State flag value
+ */
+#define FW_LA_UPDATE_STATE_SET(flags, val)	\
+	FIELD_SET(	\
+		flags, val, LA_UPDATE_STATE_BITPOS, LA_UPDATE_STATE_MASK)
+
+/*
+ *   Macro for setting the "partial" packet flag state
+ *
+ * flags      Flags to set the partial state
+ * val        Partial state value
+ */
+#define FW_LA_PARTIAL_SET(flags, val)	\
+	FIELD_SET(flags, val, LA_PARTIAL_BITPOS, LA_PARTIAL_MASK)
+
+/*
+ * Definition of the Cipher header Content Descriptor pars block
+ * Definition of the cipher processing header cd pars block.
+ * The structure is a service-specific implementation of the common
+ * 'fw_comn_req_hdr_cd_pars_s' structure.
+ */
+union fw_cipher_req_hdr_cd_pars {
+	/* LWs 2-5 */
+	struct
+	{
+		uint64_t content_desc_addr;
+		/* Address of the content descriptor */
+
+		uint16_t content_desc_resrvd1;
+		/* Content descriptor reserved field */
+
+		uint8_t content_desc_params_sz;
+		/* Size of the content descriptor parameters in quad words. These
+		 * parameters describe the session setup configuration info for the
+		 * slices that this request relies upon i.e. the configuration word and
+		 * cipher key needed by the cipher slice if there is a request for
+		 * cipher processing. */
+
+		uint8_t content_desc_hdr_resrvd2;
+		/* Content descriptor reserved field */
+
+		uint32_t content_desc_resrvd3;
+		/* Content descriptor reserved field */
+	} s;
+
+	struct
+	{
+		uint32_t cipher_key_array[FW_NUM_LONGWORDS_4];
+		/* Cipher Key Array */
+
+	} s1;
+
+};
+
+/*
+ * Definition of the Authentication header Content Descriptor pars block
+ * Definition of the authentication processing header cd pars block.
+ */
+/* Note: Authentication uses the common 'fw_comn_req_hdr_cd_pars_s'
+ * structure - similarly, it is also used by SSL3, TLS and MGF. Only cipher
+ * and cipher + authentication require service-specific implementations of
+ * the structure  */
+
+/*
+ * Definition of the Cipher + Auth header Content Descriptor pars block
+ * Definition of the cipher + auth processing header cd pars block.
+ * The structure is a service-specific implementation of the common
+ * 'fw_comn_req_hdr_cd_pars_s' structure.
+ */
+union fw_cipher_auth_req_hdr_cd_pars {
+	/* LWs 2-5 */
+	struct
+	{
+		uint64_t content_desc_addr;
+		/* Address of the content descriptor */
+
+		uint16_t content_desc_resrvd1;
+		/* Content descriptor reserved field */
+
+		uint8_t content_desc_params_sz;
+		/* Size of the content descriptor parameters in quad words. These
+		 * parameters describe the session setup configuration info for the
+		 * slices that this request relies upon i.e. the configuration word and
+		 * cipher key needed by the cipher slice if there is a request for
+		 * cipher processing. */
+
+		uint8_t content_desc_hdr_resrvd2;
+		/* Content descriptor reserved field */
+
+		uint32_t content_desc_resrvd3;
+		/* Content descriptor reserved field */
+	} s;
+
+	struct
+	{
+		uint32_t cipher_key_array[FW_NUM_LONGWORDS_4];
+		/* Cipher Key Array */
+
+	} sl;
+
+};
+
+/*
+ * Cipher content descriptor control block (header)
+ * Definition of the service-specific cipher control block header
+ * structure. This header forms part of the content descriptor
+ * block incorporating LWs 27-31, as defined by the common base
+ * parameters structure.
+ */
+struct fw_cipher_cd_ctrl_hdr
+{
+	/* LW 27 */
+	uint8_t cipher_state_sz;
+	/* State size in quad words of the cipher algorithm used in this session.
+	 * Set to zero if the algorithm doesnt provide any state */
+
+	uint8_t cipher_key_sz;
+	/* Key size in quad words of the cipher algorithm used in this session */
+
+	uint8_t cipher_cfg_offset;
+	/* Quad word offset from the content descriptor parameters address i.e.
+	 * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher
+	 * processing */
+
+	uint8_t next_curr_id;
+	/* This field combines the next and current id (each four bits) -
+	  * the next id is the most significant nibble.
+	  * Next Id:  Set to the next slice to pass the ciphered data through.
+	  * Set to FW_SLICE_DRAM_WR if the data is not to go through
+	  * any more slices after cipher.
+	  * Current Id: Initialised with the cipher  slice type */
+
+	/* LW 28 */
+	uint8_t cipher_padding_sz;
+	/* State padding size in quad words. Set to 0 if no padding is required.
+	 */
+
+	uint8_t resrvd1;
+	uint16_t resrvd2;
+	/* Reserved bytes to bring the struct to the word boundary, used by
+	 * authentication. MUST be set to 0 */
+
+	/* LWs 29-31 */
+	uint32_t resrvd3[FW_NUM_LONGWORDS_3];
+	/* Reserved bytes used by authentication. MUST be set to 0 */
+
+};
+
+/*
+ * Authentication content descriptor control block (header)
+ * Definition of the service-specific authentication control block
+ * header structure. This header forms part of the content descriptor
+ * block incorporating LWs 27-31, as defined by the common base
+ * parameters structure, the first portion of which is reserved for
+ * cipher.
+ */
+struct fw_auth_cd_ctrl_hdr
+{
+	/* LW 27 */
+	uint32_t resrvd1;
+	/* Reserved bytes, used by cipher only. MUST be set to 0 */
+
+	/* LW 28 */
+	uint8_t resrvd2;
+	/* Reserved byte, used by cipher only. MUST be set to 0 */
+
+	uint8_t hash_flags;
+	/* General flags defining the processing to perform. 0 is normal
+	 * processing
+	 * and 1 means there is a nested hash processing loop to go through */
+
+	uint8_t hash_cfg_offset;
+	/* Quad word offset from the content descriptor parameters address to the
+	 * parameters for the auth processing */
+
+	uint8_t next_curr_id;
+	/* This field combines the next and current id (each four bits) -
+	  * the next id is the most significant nibble.
+	  * Next Id:  Set to the next slice to pass the authentication data through.
+	  * Set to FW_SLICE_DRAM_WR if the data is not to go through
+	  * any more slices after authentication.
+	  * Current Id: Initialised with the authentication slice type */
+
+	/* LW 29 */
+	uint8_t resrvd3;
+	/* Now a reserved field. MUST be set to 0 */
+
+	uint8_t outer_prefix_sz;
+	/* Size in bytes of outer prefix data */
+
+	uint8_t final_sz;
+	/* Size in bytes of digest to be returned to the client if requested */
+
+	uint8_t inner_res_sz;
+	/* Size in bytes of the digest from the inner hash algorithm */
+
+	/* LW 30 */
+	uint8_t resrvd4;
+	/* Now a reserved field. MUST be set to zero. */
+
+	uint8_t inner_state1_sz;
+	/* Size in bytes of inner hash state1 data. Must be a qword multiple */
+
+	uint8_t inner_state2_offset;
+	/* Quad word offset from the content descriptor parameters pointer to the
+	 * inner state2 value */
+
+	uint8_t inner_state2_sz;
+	/* Size in bytes of inner hash state2 data. Must be a qword multiple */
+
+	/* LW 31 */
+	uint8_t outer_config_offset;
+	/* Quad word offset from the content descriptor parameters pointer to the
+	 * outer configuration information */
+
+	uint8_t outer_state1_sz;
+	/* Size in bytes of the outer state1 value */
+
+	uint8_t outer_res_sz;
+	/* Size in bytes of digest from the outer auth algorithm */
+
+	uint8_t outer_prefix_offset;
+	/* Quad word offset from the start of the inner prefix data to the outer
+	 * prefix information. Should equal the rounded inner prefix size, converted
+	 * to qwords  */
+
+};
+
+/*
+ * Cipher + Authentication content descriptor control block header
+ * Definition of both service-specific cipher + authentication control
+ * block header structures. This header forms part of the content
+ * descriptor block incorporating LWs 27-31, as defined by the common
+ * base  parameters structure.
+ */
+struct fw_cipher_auth_cd_ctrl_hdr
+{
+	/* LW 27 */
+	uint8_t cipher_state_sz;
+	/* State size in quad words of the cipher algorithm used in this session.
+	 * Set to zero if the algorithm doesnt provide any state */
+
+	uint8_t cipher_key_sz;
+	/* Key size in quad words of the cipher algorithm used in this session */
+
+	uint8_t cipher_cfg_offset;
+	/* Quad word offset from the content descriptor parameters address i.e.
+	 * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher
+	 * processing */
+
+	uint8_t next_curr_id_cipher;
+	/* This field combines the next and current id (each four bits) -
+	  * the next id is the most significant nibble.
+	  * Next Id:  Set to the next slice to pass the ciphered data through.
+	  * Set to FW_SLICE_DRAM_WR if the data is not to go through
+	  * any more slices after cipher.
+	  * Current Id: Initialised with the cipher  slice type */
+
+	/* LW 28 */
+	uint8_t cipher_padding_sz;
+	/* State padding size in quad words. Set to 0 if no padding is required.
+	 */
+
+	uint8_t hash_flags;
+	/* General flags defining the processing to perform. 0 is normal
+	 * processing
+	 * and 1 means there is a nested hash processing loop to go through */
+
+	uint8_t hash_cfg_offset;
+	/* Quad word offset from the content descriptor parameters address to the
+	 * parameters for the auth processing */
+
+	uint8_t next_curr_id_auth;
+	/* This field combines the next and current id (each four bits) -
+	  * the next id is the most significant nibble.
+	  * Next Id:  Set to the next slice to pass the authentication data through.
+	  * Set to FW_SLICE_DRAM_WR if the data is not to go through
+	  * any more slices after authentication.
+	  * Current Id: Initialised with the authentication slice type */
+
+	/* LW 29 */
+	uint8_t resrvd1;
+	/* Reserved field. MUST be set to 0 */
+
+	uint8_t outer_prefix_sz;
+	/* Size in bytes of outer prefix data */
+
+	uint8_t final_sz;
+	/* Size in bytes of digest to be returned to the client if requested */
+
+	uint8_t inner_res_sz;
+	/* Size in bytes of the digest from the inner hash algorithm */
+
+	/* LW 30 */
+	uint8_t resrvd2;
+	/* Now a reserved field. MUST be set to zero. */
+
+	uint8_t inner_state1_sz;
+	/* Size in bytes of inner hash state1 data. Must be a qword multiple */
+
+	uint8_t inner_state2_offset;
+	/* Quad word offset from the content descriptor parameters pointer to the
+	 * inner state2 value */
+
+	uint8_t inner_state2_sz;
+	/* Size in bytes of inner hash state2 data. Must be a qword multiple */
+
+	/* LW 31 */
+	uint8_t outer_config_offset;
+	/* Quad word offset from the content descriptor parameters pointer to the
+	 * outer configuration information */
+
+	uint8_t outer_state1_sz;
+	/* Size in bytes of the outer state1 value */
+
+	uint8_t outer_res_sz;
+	/* Size in bytes of digest from the outer auth algorithm */
+
+	uint8_t outer_prefix_offset;
+	/* Quad word offset from the start of the inner prefix data to the outer
+	 * prefix information. Should equal the rounded inner prefix size, converted
+	 * to qwords  */
+
+};
+
+#define FW_AUTH_HDR_FLAG_DO_NESTED 1
+/* Definition of the hash_flags bit of the auth_hdr to indicate the request
+ * requires nested hashing  */
+
+#define FW_AUTH_HDR_FLAG_NO_NESTED 0
+/* Definition of the hash_flags bit of the auth_hdr for no nested hashing
+ * required  */
+
+#define FW_CCM_GCM_AAD_SZ_MAX 240
+/* Maximum size of AAD data allowed for CCM or GCM processing. AAD data size90 -
+ * is stored in 8-bit field and must be multiple of hash block size. 240 is
+ * largest value which satisfy both requirements.AAD_SZ_MAX is in byte units  */
+
+/*
+ * request parameter #defines
+ */
+#define FW_HASH_REQUEST_PARAMETERS_OFFSET	\
+	(sizeof(fw_la_cipher_req_params_t))
+/* Offset in bytes from the start of the request parameters block to the hash
+ * (auth) request parameters  */
+
+#define FW_CIPHER_REQUEST_PARAMETERS_OFFSET (0)
+/* Offset in bytes from the start of the request parameters block to the cipher
+ * request parameters  */
+
+/*
+ * Definition of the cipher request parameters block
+ *
+ * Definition of the cipher processing request parameters block
+ * structure, which forms part of the block incorporating LWs 14-26,
+ * as defined by the common base parameters structure.
+ * Unused fields must be set to 0.
+ */
+struct fw_la_cipher_req_params {
+	/* LW 14 */
+	uint32_t cipher_offset;
+	/* Cipher offset long word. */
+
+	/* LW 15 */
+	uint32_t cipher_length;
+	/* Cipher length long word. */
+
+	/* LWs 16-19 */
+	union {
+		uint32_t cipher_IV_array[FW_NUM_LONGWORDS_4];
+		/* Cipher IV array  */
+
+		struct
+		{
+			uint64_t cipher_IV_ptr;
+			/* Cipher IV pointer or Partial State Pointer */
+
+			uint64_t resrvd1;
+			/* reserved */
+
+		} s;
+
+	} u;
+
+};
+
+/*
+ * Definition of the auth request parameters block
+ * Definition of the authentication processing request parameters block
+ * structure, which forms part of the block incorporating LWs 14-26,
+ * as defined by the common base parameters structure. Note:
+ * This structure is used by TLS only.
+ */
+struct fw_la_auth_req_params {
+	/* LW 20 */
+	uint32_t auth_off;
+	/* Byte offset from the start of packet to the auth data region */
+
+	/* LW 21 */
+	uint32_t auth_len;
+	/* Byte length of the auth data region */
+
+	/* LWs 22-23 */
+	union {
+		uint64_t auth_partial_st_prefix;
+		/* Address of the authentication partial state prefix
+		 * information */
+
+		uint64_t aad_adr;
+		/* Address of the AAD info in DRAM. Used for the CCM and GCM
+		 * protocols */
+
+	} u1;
+
+	/* LWs 24-25 */
+	uint64_t auth_res_addr;
+	/* Address of the authentication result information to validate or
+	 * the location to which the digest information can be written back to */
+
+	/* LW 26 */
+	union {
+		uint8_t inner_prefix_sz;
+		/* Size in bytes of the inner prefix data */
+
+		uint8_t aad_sz;
+		/* Size in bytes of padded AAD data to prefix to the packet for CCM
+		 *  or GCM processing */
+	} u2;
+
+	uint8_t resrvd1;
+	/* reserved */
+
+	uint8_t hash_state_sz;
+	/* Number of quad words of inner and outer hash prefix data to process
+	 * Maximum size is 240 */
+
+	uint8_t auth_res_sz;
+	/* Size in bytes of the authentication result */
+
+} __packed;
+
+/*
+ * Definition of the auth request parameters block
+ * Definition of the authentication processing request parameters block
+ * structure, which forms part of the block incorporating LWs 14-26,
+ * as defined by the common base parameters structure. Note:
+ * This structure is used by SSL3 and MGF1 only. All fields other than
+ * inner prefix/ AAD size are unused and therefore reserved.
+ */
+struct fw_la_auth_req_params_resrvd_flds {
+	/* LWs 20-25 */
+	uint32_t resrvd[FW_NUM_LONGWORDS_6];
+
+	/* LW 26 */
+	union {
+		uint8_t inner_prefix_sz;
+		/* Size in bytes of the inner prefix data */
+
+		uint8_t aad_sz;
+		/* Size in bytes of padded AAD data to prefix to the packet for CCM
+		 *  or GCM processing */
+	} u2;
+
+	uint8_t resrvd1;
+	/* reserved */
+
+	uint16_t resrvd2;
+	/* reserved */
+};
+
+/*
+ *   Definition of the shared fields within the parameter block
+ *   containing SSL, TLS or MGF information.
+ *   This structure defines the shared fields for SSL, TLS or MGF
+ *   within the parameter block incorporating LWs 14-26, as defined
+ *   by the common base parameters structure.
+ *   Unused fields must be set to 0.
+ */
+struct fw_la_key_gen_common {
+	/* LW 14 */
+	union {
+		/* SSL3 */
+		uint16_t secret_lgth_ssl;
+		/* Length of Secret information for SSL. In the case of TLS the
+		* secret is supplied in the content descriptor */
+
+		/* MGF */
+		uint16_t mask_length;
+		/* Size in bytes of the desired output mask for MGF1*/
+
+		/* TLS */
+		uint16_t secret_lgth_tls;
+		/* TLS Secret length */
+
+	} u;
+
+	union {
+		/* SSL3 */
+		struct
+		{
+			uint8_t output_lgth_ssl;
+			/* Output length */
+
+			uint8_t label_lgth_ssl;
+			/* Label length */
+
+		} s1;
+
+		/* MGF */
+		struct
+		{
+			uint8_t hash_length;
+			/* Hash length */
+
+			uint8_t seed_length;
+			/* Seed length */
+
+		} s2;
+
+		/* TLS */
+		struct
+		{
+			uint8_t output_lgth_tls;
+			/* Output length */
+
+			uint8_t label_lgth_tls;
+			/* Label length */
+
+		} s3;
+
+	} u1;
+
+	/* LW 15 */
+	union {
+		/* SSL3 */
+		uint8_t iter_count;
+		/* Iteration count used by the SSL key gen request */
+
+		/* TLS */
+		uint8_t tls_seed_length;
+		/* TLS Seed length */
+
+		uint8_t resrvd1;
+		/* Reserved field set to 0 for MGF1 */
+
+	} u2;
+
+	uint8_t resrvd2;
+	uint16_t resrvd3;
+	/* Reserved space - unused */
+
+};
+
+/*
+ *   Definition of the SSL3 request parameters block
+ *   This structure contains the the SSL3 processing request parameters
+ *   incorporating LWs 14-26, as defined by the common base
+ *   parameters structure. Unused fields must be set to 0.
+ */
+struct fw_la_ssl3_req_params {
+	/* LWs 14-15 */
+	struct fw_la_key_gen_common keygen_comn;
+	/* For other key gen processing these field holds ssl, tls or mgf
+	 *   parameters */
+
+	/* LW 16-25 */
+	uint32_t resrvd[FW_NUM_LONGWORDS_10];
+	/* Reserved */
+
+	/* LW 26 */
+	union {
+		uint8_t inner_prefix_sz;
+		/* Size in bytes of the inner prefix data */
+
+		uint8_t aad_sz;
+		/* Size in bytes of padded AAD data to prefix to the packet for CCM
+		 *  or GCM processing */
+	} u2;
+
+	uint8_t resrvd1;
+	/* reserved */
+
+	uint16_t resrvd2;
+	/* reserved */
+
+};
+
+/*
+ * Definition of the MGF request parameters block
+ * This structure contains the the MGF processing request parameters
+ * incorporating LWs 14-26, as defined by the common base parameters
+ * structure. Unused fields must be set to 0.
+ */
+struct fw_la_mgf_req_params {
+	/* LWs 14-15 */
+	struct fw_la_key_gen_common keygen_comn;
+	/* For other key gen processing these field holds ssl or mgf
+	 *   parameters */
+
+	/* LW 16-25 */
+	uint32_t resrvd[FW_NUM_LONGWORDS_10];
+	/* Reserved */
+
+	/* LW 26 */
+	union {
+		uint8_t inner_prefix_sz;
+		/* Size in bytes of the inner prefix data */
+
+		uint8_t aad_sz;
+		/* Size in bytes of padded AAD data to prefix to the packet for CCM
+		 *  or GCM processing */
+	} u2;
+
+	uint8_t resrvd1;
+	/* reserved */
+
+	uint16_t resrvd2;
+	/* reserved */
+
+};
+
+/*
+ * Definition of the TLS request parameters block
+ * This structure contains the the TLS processing request parameters
+ * incorporating LWs 14-26, as defined by the common base parameters
+ * structure. Unused fields must be set to 0.
+ */
+struct fw_la_tls_req_params {
+	/* LWs 14-15 */
+	struct fw_la_key_gen_common keygen_comn;
+	/* For other key gen processing these field holds ssl, tls or mgf
+	 *   parameters */
+
+	/* LW 16-19 */
+	uint32_t resrvd[FW_NUM_LONGWORDS_4];
+	/* Reserved */
+
+};
+
+/*
+ * Definition of the common QAT FW request middle block for TRNG.
+ * Common section of the request used across all of the services exposed
+ * by the QAT FW. Each of the services inherit these common fields. TRNG
+ * requires a specific implementation.
+ */
+struct fw_la_trng_req_mid {
+	/* LWs 6-13 */
+	uint64_t opaque_data;
+	/* Opaque data passed unmodified from the request to response messages by
+	 * firmware (fw) */
+
+	uint64_t resrvd1;
+	/* Reserved, unused for TRNG */
+
+	uint64_t dest_data_addr;
+	/* Generic definition of the destination data supplied to the QAT AE. The
+	 * common flags are used to further describe the attributes of this
+	 * field */
+
+	uint32_t resrvd2;
+	/* Reserved, unused for TRNG */
+
+	uint32_t entropy_length;
+	/* Size of the data in bytes to process. Used by the get_random
+	 * command. Set to 0 for commands that dont need a length parameter */
+
+};
+
+/*
+ * Definition of the common LA QAT FW TRNG request
+ * Definition of the TRNG processing request type
+ */
+struct fw_la_trng_req {
+	/* LWs 0-1 */
+	struct fw_comn_req_hdr comn_hdr;
+	/* Common request header */
+
+	/* LWs 2-5 */
+	union fw_comn_req_hdr_cd_pars cd_pars;
+	/* Common Request content descriptor field which points either to a
+	 * content descriptor
+	 * parameter block or contains the service-specific data itself. */
+
+	/* LWs 6-13 */
+	struct fw_la_trng_req_mid comn_mid;
+	/* TRNG request middle section - differs from the common mid-section */
+
+	/* LWs 14-26 */
+	uint32_t resrvd1[FW_NUM_LONGWORDS_13];
+
+	/* LWs 27-31 */
+	uint32_t resrvd2[FW_NUM_LONGWORDS_5];
+
+};
+
+/*
+ * Definition of the Lookaside Eagle Tail Response
+ * This is the response delivered to the ET rings by the Lookaside
+ * QAT FW service for all commands
+ */
+struct fw_la_resp {
+	/* LWs 0-1 */
+	struct fw_comn_resp_hdr comn_resp;
+	/* Common interface response format see fw.h */
+
+	/* LWs 2-3 */
+	uint64_t opaque_data;
+	/* Opaque data passed from the request to the response message */
+
+	/* LWs 4-7 */
+	uint32_t resrvd[FW_NUM_LONGWORDS_4];
+	/* Reserved */
+
+};
+
+/*
+ *   Definition of the Lookaside TRNG Test Status Structure
+ *   As an addition to FW_LA_TRNG_STATUS Pass or Fail information
+ *   in common response fields, as a response to TRNG_TEST request, Test
+ *   status, Counter for failed tests and 4 entropy counter values are
+ *   sent
+ *   Status of test status and the fail counts.
+ */
+struct fw_la_trng_test_result {
+	uint32_t test_status_info;
+	/* TRNG comparator health test status& Validity information
+	see Test Status Bit Fields below. */
+
+	uint32_t test_status_fail_count;
+	/* TRNG comparator health test status, 32bit fail counter */
+
+	uint64_t r_ent_ones_cnt;
+	/* Raw Entropy ones counter */
+
+	uint64_t r_ent_zeros_cnt;
+	/* Raw Entropy zeros counter */
+
+	uint64_t c_ent_ones_cnt;
+	/* Conditioned Entropy ones counter */
+
+	uint64_t c_ent_zeros_cnt;
+	/* Conditioned Entropy zeros counter */
+
+	uint64_t resrvd;
+	/* Reserved field must be set to zero */
+
+};
+
+/*
+ * Definition of the Lookaside SSL Key Material Input
+ * This struct defines the layout of input parameters for the
+ * SSL3 key generation (source flat buffer format)
+ */
+struct fw_la_ssl_key_material_input {
+	uint64_t seed_addr;
+	/* Pointer to seed */
+
+	uint64_t label_addr;
+	/* Pointer to label(s) */
+
+	uint64_t secret_addr;
+	/* Pointer to secret */
+
+};
+
+/*
+ * Definition of the Lookaside TLS Key Material Input
+ * This struct defines the layout of input parameters for the
+ * TLS key generation (source flat buffer format)
+ * NOTE:
+ * Secret state value (S split into S1 and S2 parts) is supplied via
+ * Content Descriptor. S1 is placed in an outer prefix buffer, and S2
+ * inside the inner prefix buffer.
+ */
+struct fw_la_tls_key_material_input {
+	uint64_t seed_addr;
+	/* Pointer to seed */
+
+	uint64_t label_addr;
+	/* Pointer to label(s) */
+
+};
+
+/*
+ * Macros using the bit position and mask to set/extract the next
+ * and current id nibbles within the next_curr_id field of the
+ * content descriptor header block, ONLY FOR CIPHER + AUTH COMBINED.
+ * Note that for cipher only or authentication only, the common macros
+ * need to be used. These are defined in the 'fw.h' common header
+ * file, as they are used by compression, cipher and authentication.
+ *
+ * cd_ctrl_hdr_t      Content descriptor control block header.
+ * val                Value of the field being set.
+ */
+/* Cipher fields within Cipher + Authentication structure */
+#define FW_CIPHER_NEXT_ID_GET(cd_ctrl_hdr_t)	\
+	((((cd_ctrl_hdr_t)->next_curr_id_cipher) &	\
+	  FW_COMN_NEXT_ID_MASK) >>	\
+	 (FW_COMN_NEXT_ID_BITPOS))
+
+#define FW_CIPHER_NEXT_ID_SET(cd_ctrl_hdr_t, val)	\
+	(cd_ctrl_hdr_t)->next_curr_id_cipher =	\
+		((((cd_ctrl_hdr_t)->next_curr_id_cipher) &	\
+		  FW_COMN_CURR_ID_MASK) |	\
+		 ((val << FW_COMN_NEXT_ID_BITPOS) &	\
+		  FW_COMN_NEXT_ID_MASK))
+
+#define FW_CIPHER_CURR_ID_GET(cd_ctrl_hdr_t)	\
+	(((cd_ctrl_hdr_t)->next_curr_id_cipher) & FW_COMN_CURR_ID_MASK)
+
+#define FW_CIPHER_CURR_ID_SET(cd_ctrl_hdr_t, val)	\
+	(cd_ctrl_hdr_t)->next_curr_id_cipher =	\
+		((((cd_ctrl_hdr_t)->next_curr_id_cipher) &	\
+		  FW_COMN_NEXT_ID_MASK) |	\
+		 ((val)&FW_COMN_CURR_ID_MASK))
+
+/* Authentication fields within Cipher + Authentication structure */
+#define FW_AUTH_NEXT_ID_GET(cd_ctrl_hdr_t)	\
+	((((cd_ctrl_hdr_t)->next_curr_id_auth) & FW_COMN_NEXT_ID_MASK) >>	\
+	 (FW_COMN_NEXT_ID_BITPOS))
+
+#define FW_AUTH_NEXT_ID_SET(cd_ctrl_hdr_t, val)	\
+	(cd_ctrl_hdr_t)->next_curr_id_auth =	\
+		((((cd_ctrl_hdr_t)->next_curr_id_auth) &	\
+		  FW_COMN_CURR_ID_MASK) |	\
+		 ((val << FW_COMN_NEXT_ID_BITPOS) &	\
+		  FW_COMN_NEXT_ID_MASK))
+
+#define FW_AUTH_CURR_ID_GET(cd_ctrl_hdr_t)	\
+	(((cd_ctrl_hdr_t)->next_curr_id_auth) & FW_COMN_CURR_ID_MASK)
+
+#define FW_AUTH_CURR_ID_SET(cd_ctrl_hdr_t, val)	\
+	(cd_ctrl_hdr_t)->next_curr_id_auth =	\
+		((((cd_ctrl_hdr_t)->next_curr_id_auth) &	\
+		  FW_COMN_NEXT_ID_MASK) |	\
+		 ((val)&FW_COMN_CURR_ID_MASK))
+
+/*  Definitions of the bits in the test_status_info of the TRNG_TEST response.
+ *  The values returned by the Lookaside service are given below
+ *  The Test result and Test Fail Count values are only valid if the Test
+ *  Results Valid (Tv) is set.
+ *
+ *  TRNG Test Status Info
+ *  + ===== + ------------------------------------------------ + --- + --- +
+ *  |  Bit  |                   31 - 2                         |  1  |  0  |
+ *  + ===== + ------------------------------------------------ + --- + --- +
+ *  | Flags |                 RESERVED = 0                     | Tv  | Ts  |
+ *  + ===== + ------------------------------------------------------------ +
+ */
+/*
+ *   Definition of the Lookaside TRNG Test Status Information received as
+ *   a part of fw_la_trng_test_result_t
+ *
+ */
+#define FW_LA_TRNG_TEST_STATUS_TS_BITPOS 0
+/* TRNG Test Result t_status field bit pos definition. */
+
+#define FW_LA_TRNG_TEST_STATUS_TS_MASK 0x1
+/* TRNG Test Result t_status field mask definition. */
+
+#define FW_LA_TRNG_TEST_STATUS_TV_BITPOS 1
+/* TRNG Test Result test results valid field bit pos definition. */
+
+#define FW_LA_TRNG_TEST_STATUS_TV_MASK 0x1
+/* TRNG Test Result test results valid field mask definition. */
+
+/*
+ *   Definition of the Lookaside TRNG test_status values.
+ *
+ *
+ */
+#define FW_LA_TRNG_TEST_STATUS_TV_VALID 1
+/* TRNG TEST Response Test Results Valid Value. */
+
+#define FW_LA_TRNG_TEST_STATUS_TV_NOT_VALID 0
+/* TRNG TEST Response Test Results are NOT Valid Value. */
+
+#define FW_LA_TRNG_TEST_STATUS_TS_NO_FAILS 1
+/* Value for TRNG Test status tests have NO FAILs Value. */
+
+#define FW_LA_TRNG_TEST_STATUS_TS_HAS_FAILS 0
+/* Value for TRNG Test status tests have one or more FAILS Value. */
+
+/*
+ *  Macro for extraction of the Test Status Field returned in the response
+ *  to TRNG TEST command.
+ *
+ * test_status        8 bit test_status value to extract the status bit
+ */
+#define FW_LA_TRNG_TEST_STATUS_TS_FLD_GET(test_status)	\
+	FIELD_GET(test_status,	\
+			      FW_LA_TRNG_TEST_STATUS_TS_BITPOS,	\
+			      FW_LA_TRNG_TEST_STATUS_TS_MASK)
+/*
+ *  Macro for extraction of the Test Results Valid Field returned in the
+ *  response to TRNG TEST command.
+ *
+ * test_status        8 bit test_status value to extract the Tests
+ *         Results valid bit
+ */
+#define FW_LA_TRNG_TEST_STATUS_TV_FLD_GET(test_status)	\
+	FIELD_GET(test_status,	\
+			      FW_LA_TRNG_TEST_STATUS_TV_BITPOS,	\
+			      FW_LA_TRNG_TEST_STATUS_TV_MASK)
+
+/*
+ * MGF Max supported input parameters
+ */
+#define FW_LA_MGF_SEED_LEN_MAX 255
+/* Maximum seed length for MGF1 request in bytes
+ * Typical values may be 48, 64, 128 bytes (or any). */
+
+#define FW_LA_MGF_MASK_LEN_MAX 65528
+/* Maximum mask length for MGF1 request in bytes
+ * Typical values may be 8 (64-bit), 16 (128-bit). MUST be quad word multiple  */
+
+/*
+ * SSL Max supported input parameters
+ */
+#define FW_LA_SSL_SECRET_LEN_MAX 512
+/* Maximum secret length for SSL3 Key Gen request (bytes)  */
+
+#define FW_LA_SSL_ITERATES_LEN_MAX 16
+/* Maximum iterations for SSL3 Key Gen request (integer)  */
+
+#define FW_LA_SSL_LABEL_LEN_MAX 136
+/* Maximum label length for SSL3 Key Gen request (bytes)  */
+
+#define FW_LA_SSL_SEED_LEN_MAX 64
+/* Maximum seed length for SSL3 Key Gen request (bytes)  */
+
+#define FW_LA_SSL_OUTPUT_LEN_MAX 248
+/* Maximum output length for SSL3 Key Gen request (bytes)  */
+
+/*
+ * TLS Max supported input parameters
+ */
+#define FW_LA_TLS_SECRET_LEN_MAX 128
+/* Maximum secret length for TLS Key Gen request (bytes)  */
+
+#define FW_LA_TLS_V1_1_SECRET_LEN_MAX 128
+/* Maximum secret length for TLS Key Gen request (bytes)  */
+
+#define FW_LA_TLS_V1_2_SECRET_LEN_MAX 64
+/* Maximum secret length for TLS Key Gen request (bytes)  */
+
+#define FW_LA_TLS_LABEL_LEN_MAX 255
+/* Maximum label length for TLS Key Gen request (bytes)  */
+
+#define FW_LA_TLS_SEED_LEN_MAX 64
+/* Maximum seed length for TLS Key Gen request (bytes)  */
+
+#define FW_LA_TLS_OUTPUT_LEN_MAX 248
+/* Maximum output length for TLS Key Gen request (bytes)  */
+
+#endif
diff --git a/sys/dev/qat/qat_hw17var.h b/sys/dev/qat/qat_hw17var.h
new file mode 100644
index 000000000000..1f3d62b86f5e
--- /dev/null
+++ b/sys/dev/qat/qat_hw17var.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qat_hw17var.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2014 Intel Corporation.
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QAT_HW17VAR_H_
+#define _DEV_PCI_QAT_HW17VAR_H_
+
+CTASSERT(CONTENT_DESC_MAX_SIZE >=
+    roundup(sizeof(union hw_cipher_algo_blk), 8) +
+    roundup(sizeof(union hw_auth_algo_blk), 8));
+
+int		qat_adm_mailbox_init(struct qat_softc *);
+int		qat_adm_mailbox_send_init(struct qat_softc *);
+int		qat_arb_init(struct qat_softc *);
+int		qat_set_ssm_wdtimer(struct qat_softc *);
+int		qat_check_slice_hang(struct qat_softc *);
+
+void		qat_hw17_crypto_setup_desc(struct qat_crypto *,
+		    struct qat_session *, struct qat_crypto_desc *);
+void		qat_hw17_crypto_setup_req_params(struct qat_crypto_bank *,
+		    struct qat_session *, struct qat_crypto_desc const *,
+		    struct qat_sym_cookie *, struct cryptop *);
+
+#endif
diff --git a/sys/dev/qat/qatreg.h b/sys/dev/qat/qatreg.h
new file mode 100644
index 000000000000..2a0be321b4f9
--- /dev/null
+++ b/sys/dev/qat/qatreg.h
@@ -0,0 +1,1582 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qatreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QATREG_H_
+#define _DEV_PCI_QATREG_H_
+
+#define	__BIT(__n)						\
+	(((uintmax_t)(__n) >= NBBY * sizeof(uintmax_t)) ? 0 :	\
+	((uintmax_t)1 << (uintmax_t)((__n) & (NBBY * sizeof(uintmax_t) - 1))))
+#define	__BITS(__m, __n)		\
+	((__BIT(MAX((__m), (__n)) + 1) - 1) ^ (__BIT(MIN((__m), (__n))) - 1))
+
+#define	__LOWEST_SET_BIT(__mask)	((((__mask) - 1) & (__mask)) ^ (__mask))
+#define	__SHIFTOUT(__x, __mask) (((__x) & (__mask)) / __LOWEST_SET_BIT(__mask))
+#define	__SHIFTIN(__x, __mask)		((__x) * __LOWEST_SET_BIT(__mask))
+
+/* Limits */
+#define MAX_NUM_AE		0x10
+#define MAX_NUM_ACCEL		6
+#define MAX_AE			0x18
+#define MAX_AE_CTX		8
+#define MAX_ARB			4
+
+#define MAX_USTORE_PER_SEG	0x8000	/* 16k * 2 */
+#define MAX_USTORE		MAX_USTORE_PER_SEG
+
+#define MAX_AE_PER_ACCEL	4	/* XXX */
+#define MAX_BANK_PER_ACCEL	16	/* XXX */
+#define MAX_RING_PER_BANK	16
+
+#define MAX_XFER_REG		128
+#define MAX_GPR_REG		128
+#define MAX_NN_REG		128
+#define MAX_LMEM_REG		1024
+#define MAX_INP_STATE		16
+#define MAX_CAM_REG		16
+#define MAX_FIFO_QWADDR		160
+
+#define MAX_EXEC_INST		100
+#define UWORD_CPYBUF_SIZE	1024	/* micro-store copy buffer (bytes) */
+#define INVLD_UWORD		0xffffffffffull	/* invalid micro-instruction */
+#define AEV2_PACKED_UWORD_BYTES	6	/* version 2 packed uword size */
+#define UWORD_MASK		0xbffffffffffull  /* micro-word mask without parity */
+
+#define AE_ALL_CTX		0xff
+
+/* PCIe configuration space parameter */
+#define NO_PCI_REG			(-1)
+#define NO_REG_OFFSET			0
+
+#define MAX_BARS			3
+
+/* Fuse Control */
+#define FUSECTL_REG			0x40
+#define FUSECTL_MASK			__BIT(31)
+
+#define LEGFUSE_REG			0x4c
+#define LEGFUSE_ACCEL_MASK_CIPHER_SLICE		__BIT(0)
+#define LEGFUSE_ACCEL_MASK_AUTH_SLICE		__BIT(1)
+#define LEGFUSE_ACCEL_MASK_PKE_SLICE		__BIT(2)
+#define LEGFUSE_ACCEL_MASK_COMPRESS_SLICE	__BIT(3)
+#define LEGFUSE_ACCEL_MASK_LZS_SLICE		__BIT(4)
+#define LEGFUSE_ACCEL_MASK_EIA3_SLICE		__BIT(5)
+#define LEGFUSE_ACCEL_MASK_SHA3_SLICE		__BIT(6)
+
+/* -------------------------------------------------------------------------- */
+/* PETRINGCSR region */
+
+/* ETR parameters */
+#define ETR_MAX_RINGS_PER_BANK	16
+
+/* ETR registers */
+#define ETR_RING_CONFIG		0x0000
+#define ETR_RING_LBASE		0x0040
+#define ETR_RING_UBASE		0x0080
+#define ETR_RING_HEAD_OFFSET	0x00C0
+#define ETR_RING_TAIL_OFFSET	0x0100
+#define ETR_RING_STAT		0x0140
+#define ETR_UO_STAT		0x0148
+#define ETR_E_STAT		0x014C
+#define ETR_NE_STAT		0x0150
+#define ETR_NF_STAT		0x0154
+#define ETR_F_STAT		0x0158
+#define ETR_C_STAT		0x015C
+#define ETR_INT_EN		0x016C
+#define ETR_INT_REG		0x0170
+#define ETR_INT_SRCSEL		0x0174
+#define ETR_INT_SRCSEL_2	0x0178
+#define ETR_INT_COL_EN		0x017C
+#define ETR_INT_COL_CTL		0x0180
+#define ETR_AP_NF_MASK		0x2000
+#define ETR_AP_NF_DEST		0x2020
+#define ETR_AP_NE_MASK		0x2040
+#define ETR_AP_NE_DEST		0x2060
+#define ETR_AP_DELAY		0x2080
+
+/* ARB registers */
+#define ARB_OFFSET			0x30000
+#define ARB_REG_SIZE			0x4
+#define ARB_WTR_SIZE			0x20
+#define ARB_REG_SLOT			0x1000
+#define ARB_WTR_OFFSET			0x010
+#define ARB_RO_EN_OFFSET		0x090
+#define ARB_WRK_2_SER_MAP_OFFSET	0x180
+#define ARB_RINGSRVARBEN_OFFSET		0x19c
+
+/* Ring Config */
+#define ETR_RING_CONFIG_LATE_HEAD_POINTER_MODE		__BIT(31)
+#define ETR_RING_CONFIG_NEAR_FULL_WM			__BITS(14, 10)
+#define ETR_RING_CONFIG_NEAR_EMPTY_WM			__BITS(9, 5)
+#define ETR_RING_CONFIG_RING_SIZE			__BITS(4, 0)
+
+#define ETR_RING_CONFIG_NEAR_WM_0		0x00
+#define ETR_RING_CONFIG_NEAR_WM_4		0x01
+#define ETR_RING_CONFIG_NEAR_WM_8		0x02
+#define ETR_RING_CONFIG_NEAR_WM_16		0x03
+#define ETR_RING_CONFIG_NEAR_WM_32		0x04
+#define ETR_RING_CONFIG_NEAR_WM_64		0x05
+#define ETR_RING_CONFIG_NEAR_WM_128		0x06
+#define ETR_RING_CONFIG_NEAR_WM_256		0x07
+#define ETR_RING_CONFIG_NEAR_WM_512		0x08
+#define ETR_RING_CONFIG_NEAR_WM_1K		0x09
+#define ETR_RING_CONFIG_NEAR_WM_2K		0x0A
+#define ETR_RING_CONFIG_NEAR_WM_4K		0x0B
+#define ETR_RING_CONFIG_NEAR_WM_8K		0x0C
+#define ETR_RING_CONFIG_NEAR_WM_16K		0x0D
+#define ETR_RING_CONFIG_NEAR_WM_32K		0x0E
+#define ETR_RING_CONFIG_NEAR_WM_64K		0x0F
+#define ETR_RING_CONFIG_NEAR_WM_128K		0x10
+#define ETR_RING_CONFIG_NEAR_WM_256K		0x11
+#define ETR_RING_CONFIG_NEAR_WM_512K		0x12
+#define ETR_RING_CONFIG_NEAR_WM_1M		0x13
+#define ETR_RING_CONFIG_NEAR_WM_2M		0x14
+#define ETR_RING_CONFIG_NEAR_WM_4M		0x15
+
+#define ETR_RING_CONFIG_SIZE_64			0x00
+#define ETR_RING_CONFIG_SIZE_128		0x01
+#define ETR_RING_CONFIG_SIZE_256		0x02
+#define ETR_RING_CONFIG_SIZE_512		0x03
+#define ETR_RING_CONFIG_SIZE_1K			0x04
+#define ETR_RING_CONFIG_SIZE_2K			0x05
+#define ETR_RING_CONFIG_SIZE_4K			0x06
+#define ETR_RING_CONFIG_SIZE_8K			0x07
+#define ETR_RING_CONFIG_SIZE_16K		0x08
+#define ETR_RING_CONFIG_SIZE_32K		0x09
+#define ETR_RING_CONFIG_SIZE_64K		0x0A
+#define ETR_RING_CONFIG_SIZE_128K		0x0B
+#define ETR_RING_CONFIG_SIZE_256K		0x0C
+#define ETR_RING_CONFIG_SIZE_512K		0x0D
+#define ETR_RING_CONFIG_SIZE_1M			0x0E
+#define ETR_RING_CONFIG_SIZE_2M			0x0F
+#define ETR_RING_CONFIG_SIZE_4M			0x10
+
+/* Default Ring Config is Nearly Full = Full and Nearly Empty = Empty */
+#define ETR_RING_CONFIG_BUILD(size)					\
+		(__SHIFTIN(ETR_RING_CONFIG_NEAR_WM_0,			\
+		    ETR_RING_CONFIG_NEAR_FULL_WM) |			\
+		__SHIFTIN(ETR_RING_CONFIG_NEAR_WM_0,			\
+		    ETR_RING_CONFIG_NEAR_EMPTY_WM) |			\
+		__SHIFTIN((size), ETR_RING_CONFIG_RING_SIZE))
+
+/* Response Ring Configuration */
+#define ETR_RING_CONFIG_BUILD_RESP(size, wm_nf, wm_ne)			\
+		(__SHIFTIN((wm_nf), ETR_RING_CONFIG_NEAR_FULL_WM) |	\
+		__SHIFTIN((wm_ne), ETR_RING_CONFIG_NEAR_EMPTY_WM) |	\
+		__SHIFTIN((size), ETR_RING_CONFIG_RING_SIZE))
+
+/* Ring Base */
+#define ETR_RING_BASE_BUILD(addr, size)					\
+		(((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size)))
+
+#define ETR_INT_REG_CLEAR_MASK	0xffff
+
+/* Initial bank Interrupt Source mask */
+#define ETR_INT_SRCSEL_MASK	0x44444444UL
+
+#define ETR_INT_SRCSEL_NEXT_OFFSET	4
+
+#define ETR_RINGS_PER_INT_SRCSEL	8
+
+#define ETR_INT_COL_CTL_ENABLE	__BIT(31)
+
+#define ETR_AP_NF_MASK_INIT	0xAAAAAAAA
+#define ETR_AP_NE_MASK_INIT	0x55555555
+
+/* Autopush destination AE bit */
+#define ETR_AP_DEST_ENABLE	__BIT(7)
+#define ETR_AP_DEST_AE		__BITS(6, 2)
+#define ETR_AP_DEST_MAILBOX	__BITS(1, 0)
+
+/* Autopush destination enable bit */
+
+/* Autopush CSR Offset */
+#define ETR_AP_BANK_OFFSET		4
+
+/* Autopush maximum rings per bank */
+#define ETR_MAX_RINGS_PER_AP_BANK		32
+
+/* Maximum mailbox per acclerator */
+#define ETR_MAX_MAILBOX_PER_ACCELERATOR		4
+
+/* Maximum AEs per mailbox */
+#define ETR_MAX_AE_PER_MAILBOX			4
+
+/* Macro to get the ring's autopush bank number */
+#define ETR_RING_AP_BANK_NUMBER(ring)	((ring) >> 5)
+
+/* Macro to get the ring's autopush mailbox number */
+#define ETR_RING_AP_MAILBOX_NUMBER(ring)		\
+    (ETR_RING_AP_BANK_NUMBER(ring) % ETR_MAX_MAILBOX_PER_ACCELERATOR)
+
+/* Macro to get the ring number in the autopush bank */
+#define ETR_RING_NUMBER_IN_AP_BANK(ring)	\
+	((ring) % ETR_MAX_RINGS_PER_AP_BANK)
+
+#define ETR_RING_EMPTY_ENTRY_SIG	(0x7F7F7F7F)
+
+/* -------------------------------------------------------------------------- */
+/* CAP_GLOBAL_CTL region */
+
+#define FCU_CTRL			0x8c0
+#define FCU_CTRL_CMD_NOOP		0
+#define FCU_CTRL_CMD_AUTH		1
+#define FCU_CTRL_CMD_LOAD		2
+#define FCU_CTRL_CMD_START		3
+#define FCU_CTRL_AE			__BITS(8, 31)
+
+#define FCU_STATUS			0x8c4
+#define FCU_STATUS_STS			__BITS(0, 2)
+#define FCU_STATUS_STS_NO		0
+#define FCU_STATUS_STS_VERI_DONE	1
+#define FCU_STATUS_STS_LOAD_DONE	2
+#define FCU_STATUS_STS_VERI_FAIL	3
+#define FCU_STATUS_STS_LOAD_FAIL	4
+#define FCU_STATUS_STS_BUSY		5
+#define FCU_STATUS_AUTHFWLD		__BIT(8)
+#define FCU_STATUS_DONE			__BIT(9)
+#define FCU_STATUS_LOADED_AE		__BITS(22, 31)
+
+#define FCU_STATUS1			0x8c8
+
+#define FCU_DRAM_ADDR_LO	0x8cc
+#define FCU_DRAM_ADDR_HI	0x8d0
+#define FCU_RAMBASE_ADDR_HI	0x8d4
+#define FCU_RAMBASE_ADDR_LO	0x8d8
+
+#define FW_AUTH_WAIT_PERIOD 10
+#define FW_AUTH_MAX_RETRY   300
+
+#define CAP_GLOBAL_CTL_BASE			0xa00
+#define CAP_GLOBAL_CTL_MISC			CAP_GLOBAL_CTL_BASE + 0x04
+#define CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN	__BIT(7)
+#define CAP_GLOBAL_CTL_RESET			CAP_GLOBAL_CTL_BASE + 0x0c
+#define CAP_GLOBAL_CTL_RESET_MASK		__BITS(31, 26)
+#define CAP_GLOBAL_CTL_RESET_ACCEL_MASK		__BITS(25, 20)
+#define CAP_GLOBAL_CTL_RESET_AE_MASK		__BITS(19, 0)
+#define CAP_GLOBAL_CTL_CLK_EN			CAP_GLOBAL_CTL_BASE + 0x50
+#define CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK	__BITS(25, 20)
+#define CAP_GLOBAL_CTL_CLK_EN_AE_MASK		__BITS(19, 0)
+
+/* -------------------------------------------------------------------------- */
+/* AE region */
+#define UPC_MASK		0x1ffff
+#define USTORE_SIZE		QAT_16K
+
+#define AE_LOCAL_AE_MASK			__BITS(31, 12)
+#define AE_LOCAL_CSR_MASK			__BITS(9, 0)
+
+/* AE_LOCAL registers */
+/* Control Store Address Register */
+#define USTORE_ADDRESS				0x000
+#define USTORE_ADDRESS_ECS			__BIT(31)
+
+#define USTORE_ECC_BIT_0	44
+#define USTORE_ECC_BIT_1	45
+#define USTORE_ECC_BIT_2	46
+#define USTORE_ECC_BIT_3	47
+#define USTORE_ECC_BIT_4	48
+#define USTORE_ECC_BIT_5	49
+#define USTORE_ECC_BIT_6	50
+
+/* Control Store Data Lower Register */
+#define USTORE_DATA_LOWER			0x004
+/* Control Store Data Upper Register */
+#define USTORE_DATA_UPPER			0x008
+/* Control Store Error Status Register */
+#define USTORE_ERROR_STATUS			0x00c
+/* Arithmetic Logic Unit Output Register */
+#define ALU_OUT					0x010
+/* Context Arbiter Control Register */
+#define CTX_ARB_CNTL				0x014
+#define CTX_ARB_CNTL_INIT			0x00000000
+/* Context Enables Register */
+#define CTX_ENABLES				0x018
+#define CTX_ENABLES_INIT			0
+#define CTX_ENABLES_INUSE_CONTEXTS		__BIT(31)
+#define CTX_ENABLES_CNTL_STORE_PARITY_ERROR	__BIT(29)
+#define CTX_ENABLES_CNTL_STORE_PARITY_ENABLE	__BIT(28)
+#define CTX_ENABLES_BREAKPOINT			__BIT(27)
+#define CTX_ENABLES_PAR_ERR			__BIT(25)
+#define CTX_ENABLES_NN_MODE			__BIT(20)
+#define CTX_ENABLES_NN_RING_EMPTY		__BIT(18)
+#define CTX_ENABLES_LMADDR_1_GLOBAL		__BIT(17)
+#define CTX_ENABLES_LMADDR_0_GLOBAL		__BIT(16)
+#define CTX_ENABLES_ENABLE			__BITS(15,8)
+
+#define CTX_ENABLES_IGNORE_W1C_MASK		\
+		(~(CTX_ENABLES_PAR_ERR |	\
+		CTX_ENABLES_BREAKPOINT |	\
+		CTX_ENABLES_CNTL_STORE_PARITY_ERROR))
+
+/* cycles from CTX_ENABLE high to CTX entering executing state */
+#define CYCLES_FROM_READY2EXE	8
+
+/* Condition Code Enable Register */
+#define CC_ENABLE				0x01c
+#define CC_ENABLE_INIT				0x2000
+
+/* CSR Context Pointer Register */
+#define CSR_CTX_POINTER				0x020
+#define CSR_CTX_POINTER_CONTEXT			__BITS(2,0)
+/* Register Error Status Register */
+#define REG_ERROR_STATUS			0x030
+/* Indirect Context Status Register */
+#define CTX_STS_INDIRECT			0x040
+#define CTX_STS_INDIRECT_UPC_INIT		0x00000000
+
+/* Active Context Status Register */
+#define ACTIVE_CTX_STATUS			0x044
+#define ACTIVE_CTX_STATUS_ABO			__BIT(31)
+#define ACTIVE_CTX_STATUS_ACNO			__BITS(0, 2)
+/* Indirect Context Signal Events Register */
+#define CTX_SIG_EVENTS_INDIRECT			0x048
+#define CTX_SIG_EVENTS_INDIRECT_INIT		0x00000001
+/* Active Context Signal Events Register */
+#define CTX_SIG_EVENTS_ACTIVE			0x04c
+/* Indirect Context Wakeup Events Register */
+#define CTX_WAKEUP_EVENTS_INDIRECT		0x050
+#define CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY	0x00000001
+#define CTX_WAKEUP_EVENTS_INDIRECT_SLEEP	0x00010000
+
+#define CTX_WAKEUP_EVENTS_INDIRECT_INIT		0x00000001
+
+/* Active Context Wakeup Events Register */
+#define CTX_WAKEUP_EVENTS_ACTIVE		0x054
+/* Indirect Context Future Count Register */
+#define CTX_FUTURE_COUNT_INDIRECT		0x058
+/* Active Context Future Count Register */
+#define CTX_FUTURE_COUNT_ACTIVE		0x05c
+/* Indirect Local Memory Address 0 Register */
+#define LM_ADDR_0_INDIRECT			0x060
+/* Active Local Memory Address 0 Register */
+#define LM_ADDR_0_ACTIVE			0x064
+/* Indirect Local Memory Address 1 Register */
+#define LM_ADDR_1_INDIRECT			0x068
+/* Active Local Memory Address 1 Register */
+#define LM_ADDR_1_ACTIVE			0x06c
+/* Byte Index Register */
+#define BYTE_INDEX				0x070
+/* Indirect Local Memory Address 0 Byte Index Register */
+#define INDIRECT_LM_ADDR_0_BYTE_INDEX		0x0e0
+/* Active Local Memory Address 0 Byte Index Register */
+#define ACTIVE_LM_ADDR_0_BYTE_INDEX		0x0e4
+/* Indirect Local Memory Address 1 Byte Index Register */
+#define INDIRECT_LM_ADDR_1_BYTE_INDEX		0x0e8
+/* Active Local Memory Address 1 Byte Index Register */
+#define ACTIVE_LM_ADDR_1_BYTE_INDEX		0x0ec
+/* Transfer Index Concatenated with Byte Index Register */
+#define T_INDEX_BYTE_INDEX			0x0f4
+/* Transfer Index Register */
+#define T_INDEX				0x074
+/* Indirect Future Count Signal Signal Register */
+#define FUTURE_COUNT_SIGNAL_INDIRECT		0x078
+/* Active Context Future Count Register */
+#define FUTURE_COUNT_SIGNAL_ACTIVE		0x07c
+/* Next Neighbor Put Register */
+#define NN_PUT					0x080
+/* Next Neighbor Get Register */
+#define NN_GET					0x084
+/* Timestamp Low Register */
+#define TIMESTAMP_LOW				0x0c0
+/* Timestamp High Register */
+#define TIMESTAMP_HIGH				0x0c4
+/* Next Neighbor Signal Register */
+#define NEXT_NEIGHBOR_SIGNAL			0x100
+/* Previous Neighbor Signal Register */
+#define PREV_NEIGHBOR_SIGNAL			0x104
+/* Same AccelEngine Signal Register */
+#define SAME_AE_SIGNAL				0x108
+/* Cyclic Redundancy Check Remainder Register */
+#define CRC_REMAINDER				0x140
+/* Profile Count Register */
+#define PROFILE_COUNT				0x144
+/* Pseudorandom Number Register */
+#define PSEUDO_RANDOM_NUMBER			0x148
+/* Signature Enable Register */
+#define SIGNATURE_ENABLE			0x150
+/* Miscellaneous Control Register */
+#define AE_MISC_CONTROL			0x160
+#define AE_MISC_CONTROL_PARITY_ENABLE		__BIT(24)
+#define AE_MISC_CONTROL_FORCE_BAD_PARITY	__BIT(23)
+#define AE_MISC_CONTROL_ONE_CTX_RELOAD		__BIT(22)
+#define AE_MISC_CONTROL_CS_RELOAD		__BITS(21, 20)
+#define AE_MISC_CONTROL_SHARE_CS		__BIT(2)
+/* Control Store Address 1 Register */
+#define USTORE_ADDRESS1			0x158
+/* Local CSR Status Register */
+#define LOCAL_CSR_STATUS					0x180
+#define LOCAL_CSR_STATUS_STATUS				0x1
+/* NULL Register */
+#define NULL_CSR				0x3fc
+
+/* AE_XFER macros */
+#define AE_XFER_AE_MASK					__BITS(31, 12)
+#define AE_XFER_CSR_MASK				__BITS(9, 2)
+
+#define AEREG_BAD_REGADDR	0xffff		/* bad register address */
+
+/* -------------------------------------------------------------------------- */
+
+#define SSMWDT(i)				((i) * 0x4000 + 0x54)
+#define SSMWDTPKE(i)				((i) * 0x4000 + 0x58)
+#define INTSTATSSM(i)				((i) * 0x4000 + 0x04)
+#define INTSTATSSM_SHANGERR			__BIT(13)
+#define PPERR(i)				((i) * 0x4000 + 0x08)
+#define PPERRID(i)				((i) * 0x4000 + 0x0C)
+#define CERRSSMSH(i)				((i) * 0x4000 + 0x10)
+#define UERRSSMSH(i)				((i) * 0x4000 + 0x18)
+#define UERRSSMSHAD(i)				((i) * 0x4000 + 0x1C)
+#define SLICEHANGSTATUS(i)			((i) * 0x4000 + 0x4C)
+#define SLICE_HANG_AUTH0_MASK			__BIT(0)
+#define SLICE_HANG_AUTH1_MASK			__BIT(1)
+#define SLICE_HANG_CPHR0_MASK			__BIT(4)
+#define SLICE_HANG_CPHR1_MASK			__BIT(5)
+#define SLICE_HANG_CMP0_MASK			__BIT(8)
+#define SLICE_HANG_CMP1_MASK			__BIT(9)
+#define SLICE_HANG_XLT0_MASK			__BIT(12)
+#define SLICE_HANG_XLT1_MASK			__BIT(13)
+#define SLICE_HANG_MMP0_MASK			__BIT(16)
+#define SLICE_HANG_MMP1_MASK			__BIT(17)
+#define SLICE_HANG_MMP2_MASK			__BIT(18)
+#define SLICE_HANG_MMP3_MASK			__BIT(19)
+#define SLICE_HANG_MMP4_MASK			__BIT(20)
+
+#define SHINTMASKSSM(i)				((i) * 0x4000 + 0x1018)
+#define ENABLE_SLICE_HANG			0x000000
+#define MAX_MMP (5)
+#define MMP_BASE(i)				((i) * 0x1000 % 0x3800)
+#define CERRSSMMMP(i, n)			((i) * 0x4000 + MMP_BASE(n) + 0x380)
+#define UERRSSMMMP(i, n)			((i) * 0x4000 + MMP_BASE(n) + 0x388)
+#define UERRSSMMMPAD(i, n)			((i) * 0x4000 + MMP_BASE(n) + 0x38C)
+
+#define CPP_CFC_ERR_STATUS			(0x30000 + 0xC04)
+#define CPP_CFC_ERR_PPID			(0x30000 + 0xC08)
+
+#define ERRSOU0					(0x3A000 + 0x00)
+#define ERRSOU1					(0x3A000 + 0x04)
+#define ERRSOU2					(0x3A000 + 0x08)
+#define ERRSOU3					(0x3A000 + 0x0C)
+#define ERRSOU4					(0x3A000 + 0xD0)
+#define ERRSOU5					(0x3A000 + 0xD8)
+#define ERRMSK0					(0x3A000 + 0x10)
+#define ERRMSK1					(0x3A000 + 0x14)
+#define ERRMSK2					(0x3A000 + 0x18)
+#define ERRMSK3					(0x3A000 + 0x1C)
+#define ERRMSK4					(0x3A000 + 0xD4)
+#define ERRMSK5					(0x3A000 + 0xDC)
+#define EMSK3_CPM0_MASK				__BIT(2)
+#define EMSK3_CPM1_MASK				__BIT(3)
+#define EMSK5_CPM2_MASK				__BIT(16)
+#define EMSK5_CPM3_MASK				__BIT(17)
+#define EMSK5_CPM4_MASK				__BIT(18)
+#define RICPPINTSTS				(0x3A000 + 0x114)
+#define RIERRPUSHID				(0x3A000 + 0x118)
+#define RIERRPULLID				(0x3A000 + 0x11C)
+
+#define TICPPINTSTS				(0x3A400 + 0x13C)
+#define TIERRPUSHID				(0x3A400 + 0x140)
+#define TIERRPULLID				(0x3A400 + 0x144)
+#define SECRAMUERR				(0x3AC00 + 0x04)
+#define SECRAMUERRAD				(0x3AC00 + 0x0C)
+#define CPPMEMTGTERR				(0x3AC00 + 0x10)
+#define ERRPPID					(0x3AC00 + 0x14)
+
+#define ADMINMSGUR				0x3a574
+#define ADMINMSGLR				0x3a578
+#define MAILBOX_BASE				0x20970
+#define MAILBOX_STRIDE				0x1000
+#define ADMINMSG_LEN				32
+
+/* -------------------------------------------------------------------------- */
+static const uint8_t mailbox_const_tab[1024] __aligned(1024) = {
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13,
+0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76,
+0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab,
+0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0,
+0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e,
+0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39,
+0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe,
+0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae,
+0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f,
+0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29,
+0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17,
+0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff,
+0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c,
+0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f,
+0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb,
+0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94,
+0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52,
+0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f,
+0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13,
+0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+/* -------------------------------------------------------------------------- */
+/* Microcode */
+
+/* Clear GPR of AE */
+static const uint64_t ae_clear_gprs_inst[] = {
+	0x0F0000C0000ull,  /* .0 l0000!val = 0 ; immed[l0000!val, 0x0] */
+	0x0F000000380ull,  /* .1 l0000!count = 128 ; immed[l0000!count, 0x80] */
+	0x0D805000011ull,  /* .2 br!=ctx[0, ctx_init#] */
+	0x0FC082C0300ull,  /* .3 local_csr_wr[nn_put, 0] */
+	0x0F0000C0300ull,  /* .4 nop */
+	0x0F0000C0300ull,  /* .5 nop */
+	0x0F0000C0300ull,  /* .6 nop */
+	0x0F0000C0300ull,  /* .7 nop */
+	0x0A0643C0000ull,  /* .8 init_nn#:alu[*n$index++, --, b, l0000!val] */
+	0x0BAC0000301ull,  /* .9 alu[l0000!count, l0000!count, -, 1] */
+	0x0D802000101ull,  /* .10 bne[init_nn#] */
+	0x0F0000C0001ull,  /* .11 l0000!indx = 0 ; immed[l0000!indx, 0x0] */
+	0x0FC066C0001ull,  /* .12 local_csr_wr[active_lm_addr_0, l0000!indx];
+			    * put indx to lm_addr */
+	0x0F0000C0300ull,  /* .13 nop */
+	0x0F0000C0300ull,  /* .14 nop */
+	0x0F0000C0300ull,  /* .15 nop */
+	0x0F000400300ull,  /* .16 l0000!count = 1024 ; immed[l0000!count, 0x400] */
+	0x0A0610C0000ull,  /* .17 init_lm#:alu[*l$index0++, --, b, l0000!val] */
+	0x0BAC0000301ull,  /* .18 alu[l0000!count, l0000!count, -, 1] */
+	0x0D804400101ull,  /* .19 bne[init_lm#] */
+	0x0A0580C0000ull,  /* .20 ctx_init#:alu[$l0000!xfers[0], --, b, l0000!val] */
+	0x0A0581C0000ull,  /* .21 alu[$l0000!xfers[1], --, b, l0000!val] */
+	0x0A0582C0000ull,  /* .22 alu[$l0000!xfers[2], --, b, l0000!val] */
+	0x0A0583C0000ull,  /* .23 alu[$l0000!xfers[3], --, b, l0000!val] */
+	0x0A0584C0000ull,  /* .24 alu[$l0000!xfers[4], --, b, l0000!val] */
+	0x0A0585C0000ull,  /* .25 alu[$l0000!xfers[5], --, b, l0000!val] */
+	0x0A0586C0000ull,  /* .26 alu[$l0000!xfers[6], --, b, l0000!val] */
+	0x0A0587C0000ull,  /* .27 alu[$l0000!xfers[7], --, b, l0000!val] */
+	0x0A0588C0000ull,  /* .28 alu[$l0000!xfers[8], --, b, l0000!val] */
+	0x0A0589C0000ull,  /* .29 alu[$l0000!xfers[9], --, b, l0000!val] */
+	0x0A058AC0000ull,  /* .30 alu[$l0000!xfers[10], --, b, l0000!val] */
+	0x0A058BC0000ull,  /* .31 alu[$l0000!xfers[11], --, b, l0000!val] */
+	0x0A058CC0000ull,  /* .32 alu[$l0000!xfers[12], --, b, l0000!val] */
+	0x0A058DC0000ull,  /* .33 alu[$l0000!xfers[13], --, b, l0000!val] */
+	0x0A058EC0000ull,  /* .34 alu[$l0000!xfers[14], --, b, l0000!val] */
+	0x0A058FC0000ull,  /* .35 alu[$l0000!xfers[15], --, b, l0000!val] */
+	0x0A05C0C0000ull,  /* .36 alu[$l0000!xfers[16], --, b, l0000!val] */
+	0x0A05C1C0000ull,  /* .37 alu[$l0000!xfers[17], --, b, l0000!val] */
+	0x0A05C2C0000ull,  /* .38 alu[$l0000!xfers[18], --, b, l0000!val] */
+	0x0A05C3C0000ull,  /* .39 alu[$l0000!xfers[19], --, b, l0000!val] */
+	0x0A05C4C0000ull,  /* .40 alu[$l0000!xfers[20], --, b, l0000!val] */
+	0x0A05C5C0000ull,  /* .41 alu[$l0000!xfers[21], --, b, l0000!val] */
+	0x0A05C6C0000ull,  /* .42 alu[$l0000!xfers[22], --, b, l0000!val] */
+	0x0A05C7C0000ull,  /* .43 alu[$l0000!xfers[23], --, b, l0000!val] */
+	0x0A05C8C0000ull,  /* .44 alu[$l0000!xfers[24], --, b, l0000!val] */
+	0x0A05C9C0000ull,  /* .45 alu[$l0000!xfers[25], --, b, l0000!val] */
+	0x0A05CAC0000ull,  /* .46 alu[$l0000!xfers[26], --, b, l0000!val] */
+	0x0A05CBC0000ull,  /* .47 alu[$l0000!xfers[27], --, b, l0000!val] */
+	0x0A05CCC0000ull,  /* .48 alu[$l0000!xfers[28], --, b, l0000!val] */
+	0x0A05CDC0000ull,  /* .49 alu[$l0000!xfers[29], --, b, l0000!val] */
+	0x0A05CEC0000ull,  /* .50 alu[$l0000!xfers[30], --, b, l0000!val] */
+	0x0A05CFC0000ull,  /* .51 alu[$l0000!xfers[31], --, b, l0000!val] */
+	0x0A0400C0000ull,  /* .52 alu[l0000!gprega[0], --, b, l0000!val] */
+	0x0B0400C0000ull,  /* .53 alu[l0000!gpregb[0], --, b, l0000!val] */
+	0x0A0401C0000ull,  /* .54 alu[l0000!gprega[1], --, b, l0000!val] */
+	0x0B0401C0000ull,  /* .55 alu[l0000!gpregb[1], --, b, l0000!val] */
+	0x0A0402C0000ull,  /* .56 alu[l0000!gprega[2], --, b, l0000!val] */
+	0x0B0402C0000ull,  /* .57 alu[l0000!gpregb[2], --, b, l0000!val] */
+	0x0A0403C0000ull,  /* .58 alu[l0000!gprega[3], --, b, l0000!val] */
+	0x0B0403C0000ull,  /* .59 alu[l0000!gpregb[3], --, b, l0000!val] */
+	0x0A0404C0000ull,  /* .60 alu[l0000!gprega[4], --, b, l0000!val] */
+	0x0B0404C0000ull,  /* .61 alu[l0000!gpregb[4], --, b, l0000!val] */
+	0x0A0405C0000ull,  /* .62 alu[l0000!gprega[5], --, b, l0000!val] */
+	0x0B0405C0000ull,  /* .63 alu[l0000!gpregb[5], --, b, l0000!val] */
+	0x0A0406C0000ull,  /* .64 alu[l0000!gprega[6], --, b, l0000!val] */
+	0x0B0406C0000ull,  /* .65 alu[l0000!gpregb[6], --, b, l0000!val] */
+	0x0A0407C0000ull,  /* .66 alu[l0000!gprega[7], --, b, l0000!val] */
+	0x0B0407C0000ull,  /* .67 alu[l0000!gpregb[7], --, b, l0000!val] */
+	0x0A0408C0000ull,  /* .68 alu[l0000!gprega[8], --, b, l0000!val] */
+	0x0B0408C0000ull,  /* .69 alu[l0000!gpregb[8], --, b, l0000!val] */
+	0x0A0409C0000ull,  /* .70 alu[l0000!gprega[9], --, b, l0000!val] */
+	0x0B0409C0000ull,  /* .71 alu[l0000!gpregb[9], --, b, l0000!val] */
+	0x0A040AC0000ull,  /* .72 alu[l0000!gprega[10], --, b, l0000!val] */
+	0x0B040AC0000ull,  /* .73 alu[l0000!gpregb[10], --, b, l0000!val] */
+	0x0A040BC0000ull,  /* .74 alu[l0000!gprega[11], --, b, l0000!val] */
+	0x0B040BC0000ull,  /* .75 alu[l0000!gpregb[11], --, b, l0000!val] */
+	0x0A040CC0000ull,  /* .76 alu[l0000!gprega[12], --, b, l0000!val] */
+	0x0B040CC0000ull,  /* .77 alu[l0000!gpregb[12], --, b, l0000!val] */
+	0x0A040DC0000ull,  /* .78 alu[l0000!gprega[13], --, b, l0000!val] */
+	0x0B040DC0000ull,  /* .79 alu[l0000!gpregb[13], --, b, l0000!val] */
+	0x0A040EC0000ull,  /* .80 alu[l0000!gprega[14], --, b, l0000!val] */
+	0x0B040EC0000ull,  /* .81 alu[l0000!gpregb[14], --, b, l0000!val] */
+	0x0A040FC0000ull,  /* .82 alu[l0000!gprega[15], --, b, l0000!val] */
+	0x0B040FC0000ull,  /* .83 alu[l0000!gpregb[15], --, b, l0000!val] */
+	0x0D81581C010ull,  /* .84 br=ctx[7, exit#] */
+	0x0E000010000ull,  /* .85 ctx_arb[kill], any */
+	0x0E000010000ull,  /* .86 exit#:ctx_arb[kill], any */
+};
+
+static const uint64_t ae_inst_4b[] = {
+	0x0F0400C0000ull,  /* .0 immed_w0[l0000!indx, 0] */
+	0x0F4400C0000ull,  /* .1 immed_w1[l0000!indx, 0] */
+	0x0F040000300ull,  /* .2 immed_w0[l0000!myvalue, 0x0] */
+	0x0F440000300ull,  /* .3 immed_w1[l0000!myvalue, 0x0] */
+	0x0FC066C0000ull,  /* .4 local_csr_wr[active_lm_addr_0, 
+	                                l0000!indx]; put indx to lm_addr */
+	0x0F0000C0300ull,  /* .5 nop */
+	0x0F0000C0300ull,  /* .6 nop */
+	0x0F0000C0300ull,  /* .7 nop */
+	0x0A021000000ull,  /* .8 alu[*l$index0++, --, b, l0000!myvalue] */
+};
+
+static const uint64_t ae_inst_1b[] = {
+	0x0F0400C0000ull,  /* .0 immed_w0[l0000!indx, 0] */
+	0x0F4400C0000ull,  /* .1 immed_w1[l0000!indx, 0] */
+	0x0F040000300ull,  /* .2 immed_w0[l0000!myvalue, 0x0] */
+	0x0F440000300ull,  /* .3 immed_w1[l0000!myvalue, 0x0] */
+	0x0FC066C0000ull,  /* .4 local_csr_wr[active_lm_addr_0, 
+	                               l0000!indx]; put indx to lm_addr */
+	0x0F0000C0300ull,  /* .5 nop */
+	0x0F0000C0300ull,  /* .6 nop */
+	0x0F0000C0300ull,  /* .7 nop */
+	0x0A000180000ull,  /* .8 alu[l0000!val, --, b, *l$index0] */
+	0x09080000200ull,  /* .9 alu_shf[l0000!myvalue, --, b, 
+	                                  l0000!myvalue,  <<24 ] */
+	0x08180280201ull,  /* .10 alu_shf[l0000!val1, --, b, l0000!val,  <<8 ] */
+	0x08080280102ull,  /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>8 ] */
+	0x0BA00100002ull,  /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */
+
+};
+
+static const uint64_t ae_inst_2b[] = {
+	0x0F0400C0000ull,  /* .0 immed_w0[l0000!indx, 0] */
+	0x0F4400C0000ull,  /* .1 immed_w1[l0000!indx, 0] */
+	0x0F040000300ull,  /* .2 immed_w0[l0000!myvalue, 0x0] */
+	0x0F440000300ull,  /* .3 immed_w1[l0000!myvalue, 0x0] */
+	0x0FC066C0000ull,  /* .4 local_csr_wr[active_lm_addr_0, 
+	                               l0000!indx]; put indx to lm_addr */
+	0x0F0000C0300ull,  /* .5 nop */
+	0x0F0000C0300ull,  /* .6 nop */
+	0x0F0000C0300ull,  /* .7 nop */
+	0x0A000180000ull,  /* .8 alu[l0000!val, --, b, *l$index0] */
+	0x09100000200ull,  /* .9 alu_shf[l0000!myvalue, --, b, 
+	                                      l0000!myvalue,  <<16 ] */
+	0x08100280201ull,  /* .10 alu_shf[l0000!val1, --, b, l0000!val,  <<16 ] */
+	0x08100280102ull,  /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>16 ] */
+	0x0BA00100002ull,  /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */
+};
+
+static const uint64_t ae_inst_3b[] = {
+	0x0F0400C0000ull,  /* .0 immed_w0[l0000!indx, 0] */
+	0x0F4400C0000ull,  /* .1 immed_w1[l0000!indx, 0] */
+	0x0F040000300ull,  /* .2 immed_w0[l0000!myvalue, 0x0] */
+	0x0F440000300ull,  /* .3 immed_w1[l0000!myvalue, 0x0] */
+	0x0FC066C0000ull,  /* .4 local_csr_wr[active_lm_addr_0, 
+	                               l0000!indx]; put indx to lm_addr */
+	0x0F0000C0300ull,  /* .5 nop */
+	0x0F0000C0300ull,  /* .6 nop */
+	0x0F0000C0300ull,  /* .7 nop */
+	0x0A000180000ull,  /* .8 alu[l0000!val, --, b, *l$index0] */
+	0x09180000200ull,  /* .9 alu_shf[l0000!myvalue, --, 
+	                                b, l0000!myvalue,  <<8 ] */
+	0x08080280201ull,  /* .10 alu_shf[l0000!val1, --, b, l0000!val,  <<24 ] */
+	0x08180280102ull,  /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>24 ] */
+	0x0BA00100002ull,  /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */
+};
+
+/* micro-instr fixup */
+#define INSERT_IMMED_GPRA_CONST(inst, const_val)		\
+		inst = (inst & 0xFFFF00C03FFull) |		\
+		((((const_val) << 12) & 0x0FF00000ull) |	\
+		(((const_val) << 10) & 0x0003FC00ull))
+#define INSERT_IMMED_GPRB_CONST(inst, const_val)		\
+		inst = (inst & 0xFFFF00FFF00ull) |		\
+		((((const_val) << 12) & 0x0FF00000ull) |	\
+		(((const_val) << 0) & 0x000000FFull))
+
+enum aereg_type {
+	AEREG_NO_DEST,		/* no destination */
+	AEREG_GPA_REL,		/* general-purpose A register under relative mode */
+	AEREG_GPA_ABS,		/* general-purpose A register under absolute mode */
+	AEREG_GPB_REL,		/* general-purpose B register under relative mode */
+	AEREG_GPB_ABS,		/* general-purpose B register under absolute mode */
+	AEREG_SR_REL,		/* sram register under relative mode */
+	AEREG_SR_RD_REL,	/* sram read register under relative mode */
+	AEREG_SR_WR_REL,	/* sram write register under relative mode */
+	AEREG_SR_ABS,		/* sram register under absolute mode */
+	AEREG_SR_RD_ABS,	/* sram read register under absolute mode */
+	AEREG_SR_WR_ABS,	/* sram write register under absolute mode */
+	AEREG_SR0_SPILL,	/* sram0 spill register */
+	AEREG_SR1_SPILL,	/* sram1 spill register */
+	AEREG_SR2_SPILL,	/* sram2 spill register */
+	AEREG_SR3_SPILL,	/* sram3 spill register */
+	AEREG_SR0_MEM_ADDR,	/* sram0 memory address register */
+	AEREG_SR1_MEM_ADDR,	/* sram1 memory address register */
+	AEREG_SR2_MEM_ADDR,	/* sram2 memory address register */
+	AEREG_SR3_MEM_ADDR,	/* sram3 memory address register */
+	AEREG_DR_REL,		/* dram register under relative mode */
+	AEREG_DR_RD_REL,	/* dram read register under relative mode */
+	AEREG_DR_WR_REL,	/* dram write register under relative mode */
+	AEREG_DR_ABS,		/* dram register under absolute mode */
+	AEREG_DR_RD_ABS,	/* dram read register under absolute mode */
+	AEREG_DR_WR_ABS,	/* dram write register under absolute mode */
+	AEREG_DR_MEM_ADDR,	/* dram memory address register */
+	AEREG_LMEM,		/* local memory */
+	AEREG_LMEM0,		/* local memory bank0 */
+	AEREG_LMEM1,		/* local memory bank1 */
+	AEREG_LMEM_SPILL,	/* local memory spill */
+	AEREG_LMEM_ADDR,	/* local memory address */
+	AEREG_NEIGH_REL,	/* next neighbour register under relative mode */
+	AEREG_NEIGH_INDX,	/* next neighbour register under index mode */
+	AEREG_SIG_REL,		/* signal register under relative mode */
+	AEREG_SIG_INDX,		/* signal register under index mode */
+	AEREG_SIG_DOUBLE,	/* signal register */
+	AEREG_SIG_SINGLE,	/* signal register */
+	AEREG_SCRATCH_MEM_ADDR,	/* scratch memory address */
+	AEREG_UMEM0,		/* ustore memory bank0 */
+	AEREG_UMEM1,		/* ustore memory bank1 */
+	AEREG_UMEM_SPILL,	/* ustore memory spill */
+	AEREG_UMEM_ADDR,	/* ustore memory address */
+	AEREG_DR1_MEM_ADDR,	/* dram segment1 address */
+	AEREG_SR0_IMPORTED,	/* sram segment0 imported data */
+	AEREG_SR1_IMPORTED,	/* sram segment1 imported data */
+	AEREG_SR2_IMPORTED,	/* sram segment2 imported data */
+	AEREG_SR3_IMPORTED,	/* sram segment3 imported data */
+	AEREG_DR_IMPORTED,	/* dram segment0 imported data */
+	AEREG_DR1_IMPORTED,	/* dram segment1 imported data */
+	AEREG_SCRATCH_IMPORTED,	/* scratch imported data */
+	AEREG_XFER_RD_ABS,	/* transfer read register under absolute mode */
+	AEREG_XFER_WR_ABS,	/* transfer write register under absolute mode */
+	AEREG_CONST_VALUE,	/* const alue */
+	AEREG_ADDR_TAKEN,	/* address taken */
+	AEREG_OPTIMIZED_AWAY,	/* optimized away */
+	AEREG_SHRAM_ADDR,	/* shared ram0 address */
+	AEREG_SHRAM1_ADDR,	/* shared ram1 address */
+	AEREG_SHRAM2_ADDR,	/* shared ram2 address */
+	AEREG_SHRAM3_ADDR,	/* shared ram3 address */
+	AEREG_SHRAM4_ADDR,	/* shared ram4 address */
+	AEREG_SHRAM5_ADDR,	/* shared ram5 address */
+	AEREG_ANY = 0xffff	/* any register */
+};
+#define AEREG_SR_INDX	AEREG_SR_ABS
+				/* sram transfer register under index mode */
+#define AEREG_DR_INDX	AEREG_DR_ABS
+				/* dram transfer register under index mode */
+#define AEREG_NEIGH_ABS	AEREG_NEIGH_INDX
+				/* next neighbor register under absolute mode */
+
+
+#define QAT_2K			0x0800
+#define QAT_4K			0x1000
+#define QAT_6K			0x1800
+#define QAT_8K			0x2000
+#define QAT_16K			0x4000
+
+#define MOF_OBJ_ID_LEN		8
+#define MOF_FID			0x00666f6d
+#define MOF_MIN_VER		0x1
+#define MOF_MAJ_VER		0x0
+#define SYM_OBJS		"SYM_OBJS"	/* symbol object string */
+#define UOF_OBJS		"UOF_OBJS"	/* uof object string */
+#define SUOF_OBJS		"SUF_OBJS"	/* suof object string */
+#define SUOF_IMAG		"SUF_IMAG"	/* suof chunk ID string */
+
+#define UOF_STRT		"UOF_STRT"	/* string table section ID */
+#define UOF_GTID		"UOF_GTID"	/* GTID section ID */
+#define UOF_IMAG		"UOF_IMAG"	/* image section ID */
+#define UOF_IMEM		"UOF_IMEM"	/* import section ID */
+#define UOF_MSEG		"UOF_MSEG"	/* memory section ID */
+
+#define CRC_POLY		0x1021
+#define CRC_WIDTH		16
+#define CRC_BITMASK(x)		(1L << (x))
+#define CRC_WIDTHMASK(width)	((((1L<<(width-1))-1L)<<1)|1L)
+
+struct mof_file_hdr {
+	u_int mfh_fid;
+	u_int mfh_csum;
+	char mfh_min_ver;
+	char mfh_maj_ver;
+	u_short mfh_reserved;
+	u_short mfh_max_chunks;
+	u_short mfh_num_chunks;
+};
+
+struct mof_file_chunk_hdr {
+	char mfch_id[MOF_OBJ_ID_LEN];
+	uint64_t mfch_offset;
+	uint64_t mfch_size;
+};
+
+struct mof_uof_hdr {
+	u_short muh_max_chunks;
+	u_short muh_num_chunks;
+	u_int muh_reserved;
+};
+
+struct mof_uof_chunk_hdr {
+	char much_id[MOF_OBJ_ID_LEN];	/* should be UOF_IMAG */
+	uint64_t much_offset;		/* uof image */
+	uint64_t much_size;		/* uof image size */
+	u_int much_name;		/* uof name string-table offset */
+	u_int much_reserved;
+};
+
+#define UOF_MAX_NUM_OF_AE	16	/* maximum number of AE */
+
+#define UOF_OBJ_ID_LEN		8	/* length of object ID */
+#define UOF_FIELD_POS_SIZE	12	/* field postion size */
+#define MIN_UOF_SIZE		24	/* minimum .uof file size */
+#define UOF_FID			0xc6c2	/* uof magic number */
+#define UOF_MIN_VER		0x11
+#define UOF_MAJ_VER		0x4
+
+struct uof_file_hdr {
+	u_short ufh_id;			/* file id and endian indicator */
+	u_short ufh_reserved1;		/* reserved for future use */
+	char ufh_min_ver;		/* file format minor version */
+	char ufh_maj_ver;		/* file format major version */
+	u_short ufh_reserved2;		/* reserved for future use */
+	u_short ufh_max_chunks;		/* max chunks in file */
+	u_short ufh_num_chunks;		/* num of actual chunks */
+};
+
+struct uof_file_chunk_hdr {
+	char ufch_id[UOF_OBJ_ID_LEN];	/* chunk identifier */
+	u_int ufch_csum;		/* chunk checksum */
+	u_int ufch_offset;		/* offset of the chunk in the file */
+	u_int ufch_size;		/* size of the chunk */
+};
+
+struct uof_obj_hdr {
+	u_int uoh_cpu_type;		/* CPU type */
+	u_short uoh_min_cpu_ver;	/* starting CPU version */
+	u_short uoh_max_cpu_ver;	/* ending CPU version */
+	short uoh_max_chunks;		/* max chunks in chunk obj */
+	short uoh_num_chunks;		/* num of actual chunks */
+	u_int uoh_reserved1;
+	u_int uoh_reserved2;
+};
+
+struct uof_chunk_hdr {
+	char uch_id[UOF_OBJ_ID_LEN];
+	u_int uch_offset;
+	u_int uch_size;
+};
+
+struct uof_str_tab {
+	u_int ust_table_len;		/* length of table */
+	u_int ust_reserved;		/* reserved for future use */
+	uint64_t ust_strings;		/* pointer to string table.
+					 * NULL terminated strings */
+};
+
+#define AE_MODE_RELOAD_CTX_SHARED	__BIT(12)
+#define AE_MODE_SHARED_USTORE		__BIT(11)
+#define AE_MODE_LMEM1			__BIT(9)
+#define AE_MODE_LMEM0			__BIT(8)
+#define AE_MODE_NN_MODE			__BITS(7, 4)
+#define AE_MODE_CTX_MODE		__BITS(3, 0)
+
+#define AE_MODE_NN_MODE_NEIGH		0
+#define AE_MODE_NN_MODE_SELF		1
+#define AE_MODE_NN_MODE_DONTCARE	0xff
+
+struct uof_image {
+	u_int ui_name;			/* image name */
+	u_int ui_ae_assigned;		/* AccelEngines assigned */
+	u_int ui_ctx_assigned;		/* AccelEngine contexts assigned */
+	u_int ui_cpu_type;		/* cpu type */
+	u_int ui_entry_address;		/* entry uaddress */
+	u_int ui_fill_pattern[2];	/* uword fill value */
+	u_int ui_reloadable_size;	/* size of reloadable ustore section */
+
+	u_char ui_sensitivity;		/*
+					 * case sensitivity: 0 = insensitive,
+					 * 1 = sensitive
+					 */
+	u_char ui_reserved;		/* reserved for future use */
+	u_short ui_ae_mode;		/*
+					 * unused<15:14>, legacyMode<13>,
+					 * reloadCtxShared<12>, sharedUstore<11>,
+					 * ecc<10>, locMem1<9>, locMem0<8>,
+					 * nnMode<7:4>, ctx<3:0>
+					 */
+
+	u_short ui_max_ver;		/* max cpu ver on which the image can run */
+	u_short ui_min_ver;		/* min cpu ver on which the image can run */
+
+	u_short ui_image_attrib;	/* image attributes */
+	u_short ui_reserved2;		/* reserved for future use */
+
+	u_short ui_num_page_regions;	/* number of page regions */
+	u_short ui_num_pages;		/* number of pages */
+
+	u_int ui_reg_tab;		/* offset to register table */
+	u_int ui_init_reg_sym_tab;	/* reg/sym init table */
+	u_int ui_sbreak_tab;		/* offset to sbreak table */
+
+	u_int ui_app_metadata;		/* application meta-data */
+	/* ui_npages of code page follows this header */
+};
+
+struct uof_obj_table {
+	u_int uot_nentries;		/* number of table entries */
+	/* uot_nentries of object follows */
+};
+
+struct uof_ae_reg {
+	u_int uar_name;			/* reg name string-table offset */
+	u_int uar_vis_name;		/* reg visible name string-table offset */
+	u_short uar_type;		/* reg type */
+	u_short uar_addr;		/* reg address */
+	u_short uar_access_mode;	/* uof_RegAccessMode_T: read/write/both/undef */
+	u_char uar_visible;		/* register visibility */
+	u_char uar_reserved1;		/* reserved for future use */
+	u_short uar_ref_count;		/* number of contiguous registers allocated */
+	u_short uar_reserved2;		/* reserved for future use */
+	u_int uar_xoid;			/* xfer order ID */
+};
+
+enum uof_value_kind {
+	UNDEF_VAL,	/* undefined value */
+	CHAR_VAL,	/* character value */
+	SHORT_VAL,	/* short value */
+	INT_VAL,	/* integer value */
+	STR_VAL,	/* string value */
+	STRTAB_VAL,	/* string table value */
+	NUM_VAL,	/* number value */
+	EXPR_VAL	/* expression value */
+};
+
+enum uof_init_type {
+	INIT_EXPR,
+	INIT_REG,
+	INIT_REG_CTX,
+	INIT_EXPR_ENDIAN_SWAP
+};
+
+struct uof_init_reg_sym {
+	u_int uirs_name;		/* symbol name */
+	char uirs_init_type;		/* 0=expr, 1=register, 2=ctxReg,
+					 * 3=expr_endian_swap */
+	char uirs_value_type;		/* EXPR_VAL, STRTAB_VAL */
+	char uirs_reg_type;		/* register type: ae_reg_type */
+	u_char uirs_ctx;		/* AE context when initType=2 */
+	u_int uirs_addr_offset;		/* reg address, or sym-value offset */
+	u_int uirs_value;		/* integer value, or expression */
+};
+
+struct uof_sbreak {
+	u_int us_page_num;		/* page number */
+	u_int us_virt_uaddr;		/* virt uaddress */
+	u_char us_sbreak_type;		/* sbreak type */
+	u_char us_reg_type;		/* register type: ae_reg_type */
+	u_short us_reserved1;		/* reserved for future use */
+	u_int us_addr_offset;		/* branch target address or offset
+					 * to be used with the reg value to
+					 * calculate the target address */
+	u_int us_reg_rddr;		/* register address */
+};
+struct uof_code_page {
+	u_int ucp_page_region;		/* page associated region */
+	u_int ucp_page_num;		/* code-page number */
+	u_char ucp_def_page;		/* default page indicator */
+	u_char ucp_reserved2;		/* reserved for future use */
+	u_short ucp_reserved1;		/* reserved for future use */
+	u_int ucp_beg_vaddr;		/* starting virtual uaddr */
+	u_int ucp_beg_paddr;		/* starting physical uaddr */
+	u_int ucp_neigh_reg_tab;	/* offset to neighbour-reg table */
+	u_int ucp_uc_var_tab;		/* offset to uC var table */
+	u_int ucp_imp_var_tab;		/* offset to import var table */
+	u_int ucp_imp_expr_tab;		/* offset to import expression table */
+	u_int ucp_code_area;		/* offset to code area */
+};
+
+struct uof_code_area {
+	u_int uca_num_micro_words;	/* number of micro words */
+	u_int uca_uword_block_tab;	/* offset to ublock table */
+};
+
+struct uof_uword_block {
+	u_int uub_start_addr;		/* start address */
+	u_int uub_num_words;		/* number of microwords */
+	u_int uub_uword_offset;		/* offset to the uwords */
+	u_int uub_reserved;		/* reserved for future use */
+};
+
+struct uof_uword_fixup {
+	u_int uuf_name;			/* offset to string table */
+	u_int uuf_uword_address;	/* micro word address */
+	u_int uuf_expr_value;		/* string table offset of expr string, or value */
+	u_char uuf_val_type;		/* VALUE_UNDEF, VALUE_NUM, VALUE_EXPR */
+	u_char uuf_value_attrs;		/* bit<0> (Scope: 0=global, 1=local),
+					 * bit<1> (init: 0=no, 1=yes) */
+	u_short uuf_reserved1;		/* reserved for future use */
+	char uuf_field_attrs[UOF_FIELD_POS_SIZE];
+					/* field pos, size, and right shift value */
+};
+
+struct uof_import_var {
+	u_int uiv_name;			/* import var name string-table offset */
+	u_char uiv_value_attrs;		/* bit<0> (Scope: 0=global),
+					 * bit<1> (init: 0=no, 1=yes) */
+	u_char uiv_reserved1;		/* reserved for future use */
+	u_short uiv_reserved2;		/* reserved for future use */
+	uint64_t uiv_value;		/* 64-bit imported value */
+};
+
+struct uof_mem_val_attr {
+	u_int umva_byte_offset;		/* byte-offset from the allocated memory */
+	u_int umva_value;		/* memory value */
+};
+
+enum uof_mem_region {
+	SRAM_REGION,	/* SRAM region */
+	DRAM_REGION,	/* DRAM0 region */
+	DRAM1_REGION,	/* DRAM1 region */
+	LMEM_REGION,	/* local memory region */
+	SCRATCH_REGION,	/* SCRATCH region */
+	UMEM_REGION,	/* micro-store region */
+	RAM_REGION,	/* RAM region */
+	SHRAM_REGION,	/* shared memory-0 region */
+	SHRAM1_REGION,	/* shared memory-1 region */
+	SHRAM2_REGION,	/* shared memory-2 region */
+	SHRAM3_REGION,	/* shared memory-3 region */
+	SHRAM4_REGION,	/* shared memory-4 region */
+	SHRAM5_REGION	/* shared memory-5 region */
+};
+
+#define UOF_SCOPE_GLOBAL	0
+#define UOF_SCOPE_LOCAL		1
+
+struct uof_init_mem {
+	u_int uim_sym_name;		/* symbol name */
+	char uim_region;		/* memory region -- uof_mem_region */
+	char uim_scope;			/* visibility scope */
+	u_short uim_reserved1;		/* reserved for future use */
+	u_int uim_addr;			/* memory address */
+	u_int uim_num_bytes;		/* number of bytes */
+	u_int uim_num_val_attr;		/* number of values attributes */
+
+	/* uim_num_val_attr of uof_mem_val_attr follows this header */
+};
+
+struct uof_var_mem_seg {
+	u_int uvms_sram_base;		/* SRAM memory segment base addr */
+	u_int uvms_sram_size;		/* SRAM segment size bytes */
+	u_int uvms_sram_alignment;	/* SRAM segment alignment bytes */
+	u_int uvms_sdram_base;		/* DRAM0 memory segment base addr */
+	u_int uvms_sdram_size;		/* DRAM0 segment size bytes */
+	u_int uvms_sdram_alignment;	/* DRAM0 segment alignment bytes */
+	u_int uvms_sdram1_base;		/* DRAM1 memory segment base addr */
+	u_int uvms_sdram1_size;		/* DRAM1 segment size bytes */
+	u_int uvms_sdram1_alignment;	/* DRAM1 segment alignment bytes */
+	u_int uvms_scratch_base;	/* SCRATCH memory segment base addr */
+	u_int uvms_scratch_size;	/* SCRATCH segment size bytes */
+	u_int uvms_scratch_alignment;	/* SCRATCH segment alignment bytes */
+};
+
+#define SUOF_OBJ_ID_LEN		8
+#define SUOF_FID		0x53554f46
+#define SUOF_MAJ_VER		0x0
+#define SUOF_MIN_VER		0x1
+#define SIMG_AE_INIT_SEQ_LEN	(50 * sizeof(unsigned long long))
+#define SIMG_AE_INSTS_LEN	(0x4000 * sizeof(unsigned long long))
+#define CSS_FWSK_MODULUS_LEN	256
+#define CSS_FWSK_EXPONENT_LEN	4
+#define CSS_FWSK_PAD_LEN	252
+#define CSS_FWSK_PUB_LEN	(CSS_FWSK_MODULUS_LEN + \
+				    CSS_FWSK_EXPONENT_LEN + \
+				    CSS_FWSK_PAD_LEN)
+#define CSS_SIGNATURE_LEN	256
+#define CSS_AE_IMG_LEN		(sizeof(struct simg_ae_mode) + \
+				    SIMG_AE_INIT_SEQ_LEN +         \
+				    SIMG_AE_INSTS_LEN)
+#define CSS_AE_SIMG_LEN		(sizeof(struct css_hdr) + \
+				    CSS_FWSK_PUB_LEN + \
+				    CSS_SIGNATURE_LEN + \
+				    CSS_AE_IMG_LEN)
+#define AE_IMG_OFFSET		(sizeof(struct css_hdr) + \
+				    CSS_FWSK_MODULUS_LEN + \
+				    CSS_FWSK_EXPONENT_LEN + \
+				    CSS_SIGNATURE_LEN)
+#define CSS_MAX_IMAGE_LEN	0x40000
+
+struct fw_auth_desc {
+	u_int fad_img_len;
+	u_int fad_reserved;
+	u_int fad_css_hdr_high;
+	u_int fad_css_hdr_low;
+	u_int fad_img_high;
+	u_int fad_img_low;
+	u_int fad_signature_high;
+	u_int fad_signature_low;
+	u_int fad_fwsk_pub_high;
+	u_int fad_fwsk_pub_low;
+	u_int fad_img_ae_mode_data_high;
+	u_int fad_img_ae_mode_data_low;
+	u_int fad_img_ae_init_data_high;
+	u_int fad_img_ae_init_data_low;
+	u_int fad_img_ae_insts_high;
+	u_int fad_img_ae_insts_low;
+};
+
+struct auth_chunk {
+	struct fw_auth_desc ac_fw_auth_desc;
+	uint64_t ac_chunk_size;
+	uint64_t ac_chunk_bus_addr;
+};
+
+enum css_fwtype {
+	CSS_AE_FIRMWARE = 0,
+	CSS_MMP_FIRMWARE = 1
+};
+
+struct css_hdr {
+	u_int css_module_type;
+	u_int css_header_len;
+	u_int css_header_ver;
+	u_int css_module_id;
+	u_int css_module_vendor;
+	u_int css_date;
+	u_int css_size;
+	u_int css_key_size;
+	u_int css_module_size;
+	u_int css_exponent_size;
+	u_int css_fw_type;
+	u_int css_reserved[21];
+};
+
+struct simg_ae_mode {
+	u_int sam_file_id;
+	u_short sam_maj_ver;
+	u_short sam_min_ver;
+	u_int sam_dev_type;
+	u_short sam_devmax_ver;
+	u_short sam_devmin_ver;
+	u_int sam_ae_mask;
+	u_int sam_ctx_enables;
+	char sam_fw_type;
+	char sam_ctx_mode;
+	char sam_nn_mode;
+	char sam_lm0_mode;
+	char sam_lm1_mode;
+	char sam_scs_mode;
+	char sam_lm2_mode;
+	char sam_lm3_mode;
+	char sam_tindex_mode;
+	u_char sam_reserved[7];
+	char sam_simg_name[256];
+	char sam_appmeta_data[256];
+};
+
+struct suof_file_hdr {
+	u_int sfh_file_id;
+	u_int sfh_check_sum;
+	char sfh_min_ver;
+	char sfh_maj_ver;
+	char sfh_fw_type;
+	char sfh_reserved;
+	u_short sfh_max_chunks;
+	u_short sfh_num_chunks;
+};
+
+struct suof_chunk_hdr {
+	char sch_chunk_id[SUOF_OBJ_ID_LEN];
+	uint64_t sch_offset;
+	uint64_t sch_size;
+};
+
+struct suof_str_tab {
+	u_int sst_tab_length;
+	u_int sst_strings;
+};
+
+struct suof_obj_hdr {
+	u_int soh_img_length;
+	u_int soh_reserved;
+};
+
+/* -------------------------------------------------------------------------- */
+/* accel */
+
+enum fw_slice {
+	FW_SLICE_NULL = 0,	/* NULL slice type */
+	FW_SLICE_CIPHER = 1,	/* CIPHER slice type */
+	FW_SLICE_AUTH = 2,	/* AUTH slice type */
+	FW_SLICE_DRAM_RD = 3,	/* DRAM_RD Logical slice type */
+	FW_SLICE_DRAM_WR = 4,	/* DRAM_WR Logical slice type */
+	FW_SLICE_COMP = 5,	/* Compression slice type */
+	FW_SLICE_XLAT = 6,	/* Translator slice type */
+	FW_SLICE_DELIMITER	/* End delimiter */
+};
+#define MAX_FW_SLICE	FW_SLICE_DELIMITER
+
+#define QAT_OPTIMAL_ALIGN_SHIFT			6
+#define QAT_OPTIMAL_ALIGN			(1 << QAT_OPTIMAL_ALIGN_SHIFT)
+
+enum hw_auth_algo {
+	HW_AUTH_ALGO_NULL = 0,		/* Null hashing */
+	HW_AUTH_ALGO_SHA1 = 1,		/* SHA1 hashing */
+	HW_AUTH_ALGO_MD5 = 2,		/* MD5 hashing */
+	HW_AUTH_ALGO_SHA224 = 3,	/* SHA-224 hashing */
+	HW_AUTH_ALGO_SHA256 = 4,	/* SHA-256 hashing */
+	HW_AUTH_ALGO_SHA384 = 5,	/* SHA-384 hashing */
+	HW_AUTH_ALGO_SHA512 = 6,	/* SHA-512 hashing */
+	HW_AUTH_ALGO_AES_XCBC_MAC = 7,	/* AES-XCBC-MAC hashing */
+	HW_AUTH_ALGO_AES_CBC_MAC = 8,	/* AES-CBC-MAC hashing */
+	HW_AUTH_ALGO_AES_F9 = 9,	/* AES F9 hashing */
+	HW_AUTH_ALGO_GALOIS_128 = 10,	/* Galois 128 bit hashing */
+	HW_AUTH_ALGO_GALOIS_64 = 11,	/* Galois 64 hashing */
+	HW_AUTH_ALGO_KASUMI_F9 = 12,	/* Kasumi F9 hashing */
+	HW_AUTH_ALGO_SNOW_3G_UIA2 = 13,	/* UIA2/SNOW_3H F9 hashing */
+	HW_AUTH_ALGO_ZUC_3G_128_EIA3 = 14,
+	HW_AUTH_RESERVED_1 = 15,
+	HW_AUTH_RESERVED_2 = 16,
+	HW_AUTH_ALGO_SHA3_256 = 17,
+	HW_AUTH_RESERVED_3 = 18,
+	HW_AUTH_ALGO_SHA3_512 = 19,
+	HW_AUTH_ALGO_DELIMITER = 20
+};
+
+enum hw_auth_mode {
+	HW_AUTH_MODE0,
+	HW_AUTH_MODE1,
+	HW_AUTH_MODE2,
+	HW_AUTH_MODE_DELIMITER
+};
+
+struct hw_auth_config {
+	uint32_t config;
+	/* Configuration used for setting up the slice */
+	uint32_t reserved;
+	/* Reserved */
+};
+
+#define HW_AUTH_CONFIG_SHA3_ALGO	__BITS(22, 23)
+#define HW_AUTH_CONFIG_SHA3_PADDING	__BIT(16)
+#define HW_AUTH_CONFIG_CMPLEN		__BITS(14, 8)
+	/* The length of the digest if the QAT is to the check*/
+#define HW_AUTH_CONFIG_MODE		__BITS(7, 4)
+#define HW_AUTH_CONFIG_ALGO		__BITS(3, 0)
+
+#define HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len)	\
+	    __SHIFTIN(mode, HW_AUTH_CONFIG_MODE) |	\
+	    __SHIFTIN(algo, HW_AUTH_CONFIG_ALGO) |	\
+	    __SHIFTIN(cmp_len, HW_AUTH_CONFIG_CMPLEN)
+
+struct hw_auth_counter {
+	uint32_t counter;		/* Counter value */
+	uint32_t reserved;		/* Reserved */
+};
+
+struct hw_auth_setup {
+	struct hw_auth_config auth_config;
+	/* Configuration word for the auth slice */
+	struct hw_auth_counter auth_counter;
+	/* Auth counter value for this request */
+};
+
+#define HW_NULL_STATE1_SZ		32
+#define HW_MD5_STATE1_SZ		16
+#define HW_SHA1_STATE1_SZ		20
+#define HW_SHA224_STATE1_SZ		32
+#define HW_SHA256_STATE1_SZ		32
+#define HW_SHA3_256_STATE1_SZ		32
+#define HW_SHA384_STATE1_SZ		64
+#define HW_SHA512_STATE1_SZ		64
+#define HW_SHA3_512_STATE1_SZ		64
+#define HW_SHA3_224_STATE1_SZ		28
+#define HW_SHA3_384_STATE1_SZ		48
+#define HW_AES_XCBC_MAC_STATE1_SZ	16
+#define HW_AES_CBC_MAC_STATE1_SZ	16
+#define HW_AES_F9_STATE1_SZ		32
+#define HW_KASUMI_F9_STATE1_SZ		16
+#define HW_GALOIS_128_STATE1_SZ		16
+#define HW_SNOW_3G_UIA2_STATE1_SZ	8
+#define HW_ZUC_3G_EIA3_STATE1_SZ	8
+#define HW_NULL_STATE2_SZ		32
+#define HW_MD5_STATE2_SZ		16
+#define HW_SHA1_STATE2_SZ		20
+#define HW_SHA224_STATE2_SZ		32
+#define HW_SHA256_STATE2_SZ		32
+#define HW_SHA3_256_STATE2_SZ		0
+#define HW_SHA384_STATE2_SZ		64
+#define HW_SHA512_STATE2_SZ		64
+#define HW_SHA3_512_STATE2_SZ		0
+#define HW_SHA3_224_STATE2_SZ		0
+#define HW_SHA3_384_STATE2_SZ		0
+#define HW_AES_XCBC_MAC_KEY_SZ		16
+#define HW_AES_CBC_MAC_KEY_SZ		16
+#define HW_AES_CCM_CBC_E_CTR0_SZ	16
+#define HW_F9_IK_SZ			16
+#define HW_F9_FK_SZ			16
+#define HW_KASUMI_F9_STATE2_SZ		(HW_F9_IK_SZ + HW_F9_FK_SZ)
+#define HW_AES_F9_STATE2_SZ		HW_KASUMI_F9_STATE2_SZ
+#define HW_SNOW_3G_UIA2_STATE2_SZ	24
+#define HW_ZUC_3G_EIA3_STATE2_SZ	32
+#define HW_GALOIS_H_SZ			16
+#define HW_GALOIS_LEN_A_SZ		8
+#define HW_GALOIS_E_CTR0_SZ		16
+
+struct hw_auth_sha512 {
+	struct hw_auth_setup inner_setup;
+	/* Inner loop configuration word for the slice */
+	uint8_t state1[HW_SHA512_STATE1_SZ];
+	/* Slice state1 variable */
+	struct hw_auth_setup outer_setup;
+	/* Outer configuration word for the slice */
+	uint8_t state2[HW_SHA512_STATE2_SZ];
+	/* Slice state2 variable */
+};
+
+union hw_auth_algo_blk {
+	struct hw_auth_sha512 max;
+	/* This is the largest possible auth setup block size */
+};
+
+enum hw_cipher_algo {
+	HW_CIPHER_ALGO_NULL = 0,		/* Null ciphering */
+	HW_CIPHER_ALGO_DES = 1,			/* DES ciphering */
+	HW_CIPHER_ALGO_3DES = 2,		/* 3DES ciphering */
+	HW_CIPHER_ALGO_AES128 = 3,		/* AES-128 ciphering */
+	HW_CIPHER_ALGO_AES192 = 4,		/* AES-192 ciphering */
+	HW_CIPHER_ALGO_AES256 = 5,		/* AES-256 ciphering */
+	HW_CIPHER_ALGO_ARC4 = 6,		/* ARC4 ciphering */
+	HW_CIPHER_ALGO_KASUMI = 7,		/* Kasumi */
+	HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8,	/* Snow_3G */
+	HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9,
+	HW_CIPHER_DELIMITER = 10		/* Delimiter type */
+};
+
+enum hw_cipher_mode {
+	HW_CIPHER_ECB_MODE = 0,		/* ECB mode */
+	HW_CIPHER_CBC_MODE = 1,		/* CBC mode */
+	HW_CIPHER_CTR_MODE = 2,		/* CTR mode */
+	HW_CIPHER_F8_MODE = 3,		/* F8 mode */
+	HW_CIPHER_XTS_MODE = 6,
+	HW_CIPHER_MODE_DELIMITER = 7	/* Delimiter type */
+};
+
+struct hw_cipher_config {
+	uint32_t val;			/* Cipher slice configuration */
+	uint32_t reserved;		/* Reserved */
+};
+
+#define CIPHER_CONFIG_CONVERT		__BIT(9)
+#define CIPHER_CONFIG_DIR		__BIT(8)
+#define CIPHER_CONFIG_MODE		__BITS(7, 4)
+#define CIPHER_CONFIG_ALGO		__BITS(3, 0)
+#define HW_CIPHER_CONFIG_BUILD(mode, algo, convert, dir)	\
+	    __SHIFTIN(mode, CIPHER_CONFIG_MODE) |		\
+	    __SHIFTIN(algo, CIPHER_CONFIG_ALGO) |		\
+	    __SHIFTIN(convert, CIPHER_CONFIG_CONVERT) |		\
+	    __SHIFTIN(dir, CIPHER_CONFIG_DIR)
+
+enum hw_cipher_dir {
+	HW_CIPHER_ENCRYPT = 0,		/* encryption is required */
+	HW_CIPHER_DECRYPT = 1,		/* decryption is required */
+};
+
+enum hw_cipher_convert {
+	HW_CIPHER_NO_CONVERT = 0,	/* no key convert is required*/
+	HW_CIPHER_KEY_CONVERT = 1,	/* key conversion is required*/
+};
+
+#define CIPHER_MODE_F8_KEY_SZ_MULT	2
+#define CIPHER_MODE_XTS_KEY_SZ_MULT	2
+
+#define HW_DES_BLK_SZ			8
+#define HW_3DES_BLK_SZ			8
+#define HW_NULL_BLK_SZ			8
+#define HW_AES_BLK_SZ			16
+#define HW_KASUMI_BLK_SZ		8
+#define HW_SNOW_3G_BLK_SZ		8
+#define HW_ZUC_3G_BLK_SZ		8
+#define HW_NULL_KEY_SZ			256
+#define HW_DES_KEY_SZ			8
+#define HW_3DES_KEY_SZ			24
+#define HW_AES_128_KEY_SZ		16
+#define HW_AES_192_KEY_SZ		24
+#define HW_AES_256_KEY_SZ		32
+#define HW_AES_128_F8_KEY_SZ		(HW_AES_128_KEY_SZ *	\
+					    CIPHER_MODE_F8_KEY_SZ_MULT)
+#define HW_AES_192_F8_KEY_SZ		(HW_AES_192_KEY_SZ *	\
+					    CIPHER_MODE_F8_KEY_SZ_MULT)
+#define HW_AES_256_F8_KEY_SZ		(HW_AES_256_KEY_SZ *	\
+					    CIPHER_MODE_F8_KEY_SZ_MULT)
+#define HW_AES_128_XTS_KEY_SZ		(HW_AES_128_KEY_SZ *	\
+					    CIPHER_MODE_XTS_KEY_SZ_MULT)
+#define HW_AES_256_XTS_KEY_SZ		(HW_AES_256_KEY_SZ *	\
+					    CIPHER_MODE_XTS_KEY_SZ_MULT)
+#define HW_KASUMI_KEY_SZ		16
+#define HW_KASUMI_F8_KEY_SZ		(HW_KASUMI_KEY_SZ *	\
+					    CIPHER_MODE_F8_KEY_SZ_MULT)
+#define HW_AES_128_XTS_KEY_SZ		(HW_AES_128_KEY_SZ *	\
+					    CIPHER_MODE_XTS_KEY_SZ_MULT)
+#define HW_AES_256_XTS_KEY_SZ		(HW_AES_256_KEY_SZ *	\
+					    CIPHER_MODE_XTS_KEY_SZ_MULT)
+#define HW_ARC4_KEY_SZ			256
+#define HW_SNOW_3G_UEA2_KEY_SZ		16
+#define HW_SNOW_3G_UEA2_IV_SZ		16
+#define HW_ZUC_3G_EEA3_KEY_SZ		16
+#define HW_ZUC_3G_EEA3_IV_SZ		16
+#define HW_MODE_F8_NUM_REG_TO_CLEAR	2
+
+struct hw_cipher_aes256_f8 {
+	struct hw_cipher_config cipher_config;
+	/* Cipher configuration word for the slice set to
+	 * AES-256 and the F8 mode */
+	uint8_t key[HW_AES_256_F8_KEY_SZ];
+	/* Cipher key */
+};
+
+union hw_cipher_algo_blk {
+	struct hw_cipher_aes256_f8 max;		/* AES-256 F8 Cipher */
+	/* This is the largest possible cipher setup block size */
+};
+
+struct flat_buffer_desc {
+	uint32_t data_len_in_bytes;
+	uint32_t reserved;
+	uint64_t phy_buffer;
+};
+
+struct buffer_list_desc {
+	uint64_t resrvd;
+	uint32_t num_buffers;
+	uint32_t reserved;
+};
+
+/* -------------------------------------------------------------------------- */
+/* look aside */
+
+enum fw_la_cmd_id {
+	FW_LA_CMD_CIPHER,			/* Cipher Request */
+	FW_LA_CMD_AUTH,			/* Auth Request */
+	FW_LA_CMD_CIPHER_HASH,		/* Cipher-Hash Request */
+	FW_LA_CMD_HASH_CIPHER,		/* Hash-Cipher Request */
+	FW_LA_CMD_TRNG_GET_RANDOM,		/* TRNG Get Random Request */
+	FW_LA_CMD_TRNG_TEST,		/* TRNG Test Request */
+	FW_LA_CMD_SSL3_KEY_DERIVE,		/* SSL3 Key Derivation Request */
+	FW_LA_CMD_TLS_V1_1_KEY_DERIVE,	/* TLS Key Derivation Request */
+	FW_LA_CMD_TLS_V1_2_KEY_DERIVE,	/* TLS Key Derivation Request */
+	FW_LA_CMD_MGF1,			/* MGF1 Request */
+	FW_LA_CMD_AUTH_PRE_COMP,		/* Auth Pre-Compute Request */
+#if 0 /* incompatible between qat 1.5 and 1.7 */
+	FW_LA_CMD_CIPHER_CIPHER,		/* Cipher-Cipher Request */
+	FW_LA_CMD_HASH_HASH,		/* Hash-Hash Request */
+	FW_LA_CMD_CIPHER_PRE_COMP,		/* Auth Pre-Compute Request */
+#endif
+	FW_LA_CMD_DELIMITER,		/* Delimiter type */
+};
+
+#endif
diff --git a/sys/dev/qat/qatvar.h b/sys/dev/qat/qatvar.h
new file mode 100644
index 000000000000..10bc2833ca7d
--- /dev/null
+++ b/sys/dev/qat/qatvar.h
@@ -0,0 +1,1073 @@
+/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
+/*	$NetBSD: qatvar.h,v 1.2 2020/03/14 18:08:39 ad Exp $	*/
+
+/*
+ * Copyright (c) 2019 Internet Initiative Japan, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* $FreeBSD$ */
+
+#ifndef _DEV_PCI_QATVAR_H_
+#define _DEV_PCI_QATVAR_H_
+
+#include <sys/counter.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptodev.h>
+
+#define QAT_NSYMREQ	256
+#define QAT_NSYMCOOKIE	((QAT_NSYMREQ * 2 + 1) * 2)
+#define QAT_NASYMREQ	64
+#define QAT_BATCH_SUBMIT_FREE_SPACE	2
+
+#define QAT_EV_NAME_SIZE		32
+#define QAT_RING_NAME_SIZE		32
+
+#define QAT_MAXSEG			32	/* max segments for sg dma */
+#define QAT_MAXLEN			65535	/* IP_MAXPACKET */
+
+#define QAT_HB_INTERVAL			500	/* heartbeat msec */
+#define QAT_SSM_WDT			100
+
+enum qat_chip_type {
+	QAT_CHIP_C2XXX = 0,	/* NanoQAT: Atom C2000 */
+	QAT_CHIP_C2XXX_IOV,	
+	QAT_CHIP_C3XXX,		/* Atom C3000 */
+	QAT_CHIP_C3XXX_IOV,
+	QAT_CHIP_C62X,
+	QAT_CHIP_C62X_IOV,
+	QAT_CHIP_D15XX,
+	QAT_CHIP_D15XX_IOV,
+	QAT_CHIP_DH895XCC,
+	QAT_CHIP_DH895XCC_IOV,
+};
+
+enum qat_sku {
+	QAT_SKU_UNKNOWN = 0,
+	QAT_SKU_1,
+	QAT_SKU_2,
+	QAT_SKU_3,
+	QAT_SKU_4,
+	QAT_SKU_VF,
+};
+
+enum qat_ae_status {
+	QAT_AE_ENABLED = 1,
+	QAT_AE_ACTIVE,
+	QAT_AE_DISABLED
+};
+
+#define TIMEOUT_AE_RESET	100
+#define TIMEOUT_AE_CHECK	10000
+#define TIMEOUT_AE_CSR		500
+#define AE_EXEC_CYCLE		20
+
+#define QAT_UOF_MAX_PAGE		1
+#define QAT_UOF_MAX_PAGE_REGION		1
+
+struct qat_dmamem {
+	bus_dma_tag_t qdm_dma_tag;
+	bus_dmamap_t qdm_dma_map;
+	bus_size_t qdm_dma_size;
+	bus_dma_segment_t qdm_dma_seg;
+	void *qdm_dma_vaddr;
+};
+
+/* Valid internal ring size values */
+#define QAT_RING_SIZE_128 0x01
+#define QAT_RING_SIZE_256 0x02
+#define QAT_RING_SIZE_512 0x03
+#define QAT_RING_SIZE_4K 0x06
+#define QAT_RING_SIZE_16K 0x08
+#define QAT_RING_SIZE_4M 0x10
+#define QAT_MIN_RING_SIZE QAT_RING_SIZE_128
+#define QAT_MAX_RING_SIZE QAT_RING_SIZE_4M
+#define QAT_DEFAULT_RING_SIZE QAT_RING_SIZE_16K
+
+/* Valid internal msg size values */
+#define QAT_MSG_SIZE_32 0x01
+#define QAT_MSG_SIZE_64 0x02
+#define QAT_MSG_SIZE_128 0x04
+#define QAT_MIN_MSG_SIZE QAT_MSG_SIZE_32
+#define QAT_MAX_MSG_SIZE QAT_MSG_SIZE_128
+
+/* Size to bytes conversion macros for ring and msg size values */
+#define QAT_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5)
+#define QAT_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5)
+#define QAT_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7)
+#define QAT_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7)
+
+/* Minimum ring bufer size for memory allocation */
+#define QAT_RING_SIZE_BYTES_MIN(SIZE) \
+	((SIZE < QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K)) ? \
+		QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K) : SIZE)
+#define QAT_RING_SIZE_MODULO(SIZE) (SIZE + 0x6)
+#define QAT_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \
+				SIZE) & ~0x4)
+/* Max outstanding requests */
+#define QAT_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \
+	((((1 << (RING_SIZE - 1)) << 3) >> QAT_SIZE_TO_POW(MSG_SIZE)) - 1)
+
+#define QAT_RING_PATTERN		0x7f
+
+struct qat_softc;
+
+typedef int (*qat_cb_t)(struct qat_softc *, void *, void *);
+
+struct qat_ring {
+	struct mtx qr_ring_mtx;   /* Lock per ring */
+	bool qr_need_wakeup;
+	void *qr_ring_vaddr;
+	uint32_t * volatile qr_inflight;	/* tx/rx shared */
+	uint32_t qr_head;
+	uint32_t qr_tail;
+	uint8_t qr_msg_size;
+	uint8_t qr_ring_size;
+	uint32_t qr_ring;	/* ring number in bank */
+	uint32_t qr_bank;	/* bank number in device */
+	uint32_t qr_ring_id;
+	uint32_t qr_ring_mask;
+	qat_cb_t qr_cb;
+	void *qr_cb_arg;
+	struct qat_dmamem qr_dma;
+	bus_addr_t qr_ring_paddr;
+
+	const char *qr_name;
+};
+
+struct qat_bank {
+	struct qat_softc *qb_sc;	/* back pointer to softc */
+	uint32_t qb_intr_mask;		/* current interrupt mask */
+	uint32_t qb_allocated_rings;	/* current allocated ring bitfiled */
+	uint32_t qb_coalescing_time;	/* timer in nano sec, 0: disabled */
+#define COALESCING_TIME_INTERVAL_DEFAULT	10000
+#define COALESCING_TIME_INTERVAL_MIN		500
+#define COALESCING_TIME_INTERVAL_MAX		0xfffff
+	uint32_t qb_bank;		/* bank index */
+	struct mtx qb_bank_mtx;
+	struct resource *qb_ih;
+	void *qb_ih_cookie;
+
+	struct qat_ring qb_et_rings[MAX_RING_PER_BANK];
+
+};
+
+struct qat_ap_bank {
+	uint32_t qab_nf_mask;
+	uint32_t qab_nf_dest;
+	uint32_t qab_ne_mask;
+	uint32_t qab_ne_dest;
+};
+
+struct qat_ae_page {
+	struct qat_ae_page *qap_next;
+	struct qat_uof_page *qap_page;
+	struct qat_ae_region *qap_region;
+	u_int qap_flags;
+};
+
+#define QAT_AE_PAGA_FLAG_WAITING	(1 << 0)
+
+struct qat_ae_region {
+	struct qat_ae_page *qar_loaded_page;
+	STAILQ_HEAD(, qat_ae_page) qar_waiting_pages;
+};
+
+struct qat_ae_slice {
+	u_int qas_assigned_ctx_mask;
+	struct qat_ae_region qas_regions[QAT_UOF_MAX_PAGE_REGION];
+	struct qat_ae_page qas_pages[QAT_UOF_MAX_PAGE];
+	struct qat_ae_page *qas_cur_pages[MAX_AE_CTX];
+	struct qat_uof_image *qas_image;
+};
+
+#define QAT_AE(sc, ae)			\
+		((sc)->sc_ae[ae])
+
+struct qat_ae {
+	u_int qae_state;		/* AE state */
+	u_int qae_ustore_size;		/* free micro-store address */
+	u_int qae_free_addr;		/* free micro-store address */
+	u_int qae_free_size;		/* free micro-store size */
+	u_int qae_live_ctx_mask;	/* live context mask */
+	u_int qae_ustore_dram_addr;	/* mirco-store DRAM address */
+	u_int qae_reload_size;		/* reloadable code size */
+
+	/* aefw */
+	u_int qae_num_slices;
+	struct qat_ae_slice qae_slices[MAX_AE_CTX];
+	u_int qae_reloc_ustore_dram;	/* reloadable ustore-dram address */
+	u_int qae_effect_ustore_size;	/* effective AE ustore size */
+	u_int qae_shareable_ustore;
+};
+
+struct qat_mof {
+	void *qmf_sym;			/* SYM_OBJS in sc_fw_mof */
+	size_t qmf_sym_size;
+	void *qmf_uof_objs;		/* UOF_OBJS in sc_fw_mof */
+	size_t qmf_uof_objs_size;
+	void *qmf_suof_objs;		/* SUOF_OBJS in sc_fw_mof */
+	size_t qmf_suof_objs_size;
+};
+
+struct qat_ae_batch_init {
+	u_int qabi_ae;
+	u_int qabi_addr;
+	u_int *qabi_value;
+	u_int qabi_size;
+	STAILQ_ENTRY(qat_ae_batch_init) qabi_next;
+};
+
+STAILQ_HEAD(qat_ae_batch_init_list, qat_ae_batch_init);
+
+/* overwritten struct uof_uword_block */
+struct qat_uof_uword_block {
+	u_int quub_start_addr;		/* start address */
+	u_int quub_num_words;		/* number of microwords */
+	uint64_t quub_micro_words;	/* pointer to the uwords */
+};
+
+struct qat_uof_page {
+	u_int qup_page_num;		/* page number */
+	u_int qup_def_page;		/* default page */
+	u_int qup_page_region;		/* region of page */
+	u_int qup_beg_vaddr;		/* begin virtual address */ 
+	u_int qup_beg_paddr;		/* begin physical address */
+
+	u_int qup_num_uc_var;		/* num of uC var in array */
+	struct uof_uword_fixup *qup_uc_var;
+					/* array of import variables */
+	u_int qup_num_imp_var;		/* num of import var in array */
+	struct uof_import_var *qup_imp_var;
+					/* array of import variables */
+	u_int qup_num_imp_expr;		/* num of import expr in array */
+	struct uof_uword_fixup *qup_imp_expr;
+					/* array of import expressions */
+	u_int qup_num_neigh_reg;	/* num of neigh-reg in array */
+	struct uof_uword_fixup *qup_neigh_reg;
+					/* array of neigh-reg assignments */
+	u_int qup_num_micro_words;	/* number of microwords in the seg */
+
+	u_int qup_num_uw_blocks;	/* number of uword blocks */
+	struct qat_uof_uword_block *qup_uw_blocks;
+					/* array of uword blocks */
+};
+
+struct qat_uof_image {
+	struct uof_image *qui_image;		/* image pointer */
+	struct qat_uof_page qui_pages[QAT_UOF_MAX_PAGE];
+						/* array of pages */
+
+	u_int qui_num_ae_reg;			/* num of registers */
+	struct uof_ae_reg *qui_ae_reg;		/* array of registers */
+
+	u_int qui_num_init_reg_sym;		/* num of reg/sym init values */
+	struct uof_init_reg_sym *qui_init_reg_sym;
+					/* array of reg/sym init values */
+
+	u_int qui_num_sbreak;			/* num of sbreak values */
+	struct qui_sbreak *qui_sbreak;		/* array of sbreak values */
+
+	u_int qui_num_uwords_used;
+				/* highest uword addressreferenced + 1 */
+};
+
+struct qat_aefw_uof {
+	size_t qafu_size;			/* uof size */
+	struct uof_obj_hdr *qafu_obj_hdr;	/* UOF_OBJS */
+
+	void *qafu_str_tab;
+	size_t qafu_str_tab_size;
+
+	u_int qafu_num_init_mem;
+	struct uof_init_mem *qafu_init_mem;
+	size_t qafu_init_mem_size;
+
+	struct uof_var_mem_seg *qafu_var_mem_seg;
+
+	struct qat_ae_batch_init_list qafu_lm_init[MAX_AE];
+	size_t qafu_num_lm_init[MAX_AE];
+	size_t qafu_num_lm_init_inst[MAX_AE];
+
+	u_int qafu_num_imgs;			/* number of uof image */
+	struct qat_uof_image qafu_imgs[MAX_NUM_AE * MAX_AE_CTX];
+						/* uof images */
+};
+
+#define QAT_SERVICE_CRYPTO_A		(1 << 0)
+#define QAT_SERVICE_CRYPTO_B		(1 << 1)
+
+struct qat_admin_rings {
+	uint32_t qadr_active_aes_per_accel;
+	uint8_t qadr_srv_mask[MAX_AE_PER_ACCEL];
+
+	struct qat_dmamem qadr_dma;
+	struct fw_init_ring_table *qadr_master_ring_tbl;
+	struct fw_init_ring_table *qadr_cya_ring_tbl;
+	struct fw_init_ring_table *qadr_cyb_ring_tbl;
+
+	struct qat_ring *qadr_admin_tx;
+	struct qat_ring *qadr_admin_rx;
+};
+
+struct qat_accel_init_cb {
+	int qaic_status;
+};
+
+struct qat_admin_comms {
+	struct qat_dmamem qadc_dma;
+	struct qat_dmamem qadc_const_tbl_dma;
+	struct qat_dmamem qadc_hb_dma;
+};
+
+#define QAT_PID_MINOR_REV 0xf
+#define QAT_PID_MAJOR_REV (0xf << 4)
+
+struct qat_suof_image {
+	char *qsi_simg_buf;
+	u_long qsi_simg_len;
+	char *qsi_css_header;
+	char *qsi_css_key;
+	char *qsi_css_signature;
+	char *qsi_css_simg;
+	u_long qsi_simg_size;
+	u_int qsi_ae_num;
+	u_int qsi_ae_mask;
+	u_int qsi_fw_type;
+	u_long qsi_simg_name;
+	u_long qsi_appmeta_data;
+	struct qat_dmamem qsi_dma;
+};
+
+struct qat_aefw_suof {
+	u_int qafs_file_id;
+	u_int qafs_check_sum;
+	char qafs_min_ver;
+	char qafs_maj_ver;
+	char qafs_fw_type;
+	char *qafs_suof_buf;
+	u_int qafs_suof_size;
+	char *qafs_sym_str;
+	u_int qafs_sym_size;
+	u_int qafs_num_simgs;
+	struct qat_suof_image *qafs_simg;
+};
+
+enum qat_sym_hash_algorithm {
+	QAT_SYM_HASH_NONE = 0,
+	QAT_SYM_HASH_MD5 = 1,
+	QAT_SYM_HASH_SHA1 = 2,
+	QAT_SYM_HASH_SHA224 = 3,
+	QAT_SYM_HASH_SHA256 = 4,
+	QAT_SYM_HASH_SHA384 = 5,
+	QAT_SYM_HASH_SHA512 = 6,
+	QAT_SYM_HASH_AES_XCBC = 7,
+	QAT_SYM_HASH_AES_CCM = 8,
+	QAT_SYM_HASH_AES_GCM = 9,
+	QAT_SYM_HASH_KASUMI_F9 = 10,
+	QAT_SYM_HASH_SNOW3G_UIA2 = 11,
+	QAT_SYM_HASH_AES_CMAC = 12,
+	QAT_SYM_HASH_AES_GMAC = 13,
+	QAT_SYM_HASH_AES_CBC_MAC = 14,
+};
+
+#define QAT_HASH_MD5_BLOCK_SIZE			64
+#define QAT_HASH_MD5_DIGEST_SIZE		16
+#define QAT_HASH_MD5_STATE_SIZE			16
+#define QAT_HASH_SHA1_BLOCK_SIZE		64
+#define QAT_HASH_SHA1_DIGEST_SIZE		20
+#define QAT_HASH_SHA1_STATE_SIZE		20
+#define QAT_HASH_SHA224_BLOCK_SIZE		64
+#define QAT_HASH_SHA224_DIGEST_SIZE		28
+#define QAT_HASH_SHA224_STATE_SIZE		32
+#define QAT_HASH_SHA256_BLOCK_SIZE		64
+#define QAT_HASH_SHA256_DIGEST_SIZE		32
+#define QAT_HASH_SHA256_STATE_SIZE		32
+#define QAT_HASH_SHA384_BLOCK_SIZE		128
+#define QAT_HASH_SHA384_DIGEST_SIZE		48
+#define QAT_HASH_SHA384_STATE_SIZE		64
+#define QAT_HASH_SHA512_BLOCK_SIZE		128
+#define QAT_HASH_SHA512_DIGEST_SIZE		64
+#define QAT_HASH_SHA512_STATE_SIZE		64
+#define QAT_HASH_XCBC_PRECOMP_KEY_NUM		3
+#define QAT_HASH_XCBC_MAC_BLOCK_SIZE		16
+#define QAT_HASH_XCBC_MAC_128_DIGEST_SIZE	16
+#define QAT_HASH_CMAC_BLOCK_SIZE		16
+#define QAT_HASH_CMAC_128_DIGEST_SIZE		16
+#define QAT_HASH_AES_CCM_BLOCK_SIZE		16
+#define QAT_HASH_AES_CCM_DIGEST_SIZE		16
+#define QAT_HASH_AES_GCM_BLOCK_SIZE		16
+#define QAT_HASH_AES_GCM_DIGEST_SIZE		16
+#define QAT_HASH_AES_GCM_STATE_SIZE		16
+#define QAT_HASH_KASUMI_F9_BLOCK_SIZE		8
+#define QAT_HASH_KASUMI_F9_DIGEST_SIZE		4
+#define QAT_HASH_SNOW3G_UIA2_BLOCK_SIZE		8
+#define QAT_HASH_SNOW3G_UIA2_DIGEST_SIZE	4
+#define QAT_HASH_AES_CBC_MAC_BLOCK_SIZE		16
+#define QAT_HASH_AES_CBC_MAC_DIGEST_SIZE	16
+#define QAT_HASH_AES_GCM_ICV_SIZE_8		8
+#define QAT_HASH_AES_GCM_ICV_SIZE_12		12
+#define QAT_HASH_AES_GCM_ICV_SIZE_16		16
+#define QAT_HASH_AES_CCM_ICV_SIZE_MIN		4
+#define QAT_HASH_AES_CCM_ICV_SIZE_MAX		16
+#define QAT_HASH_IPAD_BYTE			0x36
+#define QAT_HASH_OPAD_BYTE			0x5c
+#define QAT_HASH_IPAD_4_BYTES			0x36363636
+#define QAT_HASH_OPAD_4_BYTES			0x5c5c5c5c
+#define QAT_HASH_KASUMI_F9_KEY_MODIFIER_4_BYTES	0xAAAAAAAA
+
+#define QAT_SYM_XCBC_STATE_SIZE		((QAT_HASH_XCBC_MAC_BLOCK_SIZE) * 3)
+#define QAT_SYM_CMAC_STATE_SIZE		((QAT_HASH_CMAC_BLOCK_SIZE) * 3)
+
+struct qat_sym_hash_alg_info {
+	uint32_t qshai_digest_len;		/* Digest length in bytes */
+	uint32_t qshai_block_len;		/* Block length in bytes */
+	uint32_t qshai_state_size;		/* size of above state in bytes */
+	const uint8_t *qshai_init_state;	/* Initial state */
+
+	const struct auth_hash *qshai_sah;	/* software auth hash */
+	uint32_t qshai_state_offset;		/* offset to state in *_CTX */
+	uint32_t qshai_state_word;
+};
+
+struct qat_sym_hash_qat_info {
+	uint32_t qshqi_algo_enc;	/* QAT Algorithm encoding */
+	uint32_t qshqi_auth_counter;	/* Counter value for Auth */
+	uint32_t qshqi_state1_len;	/* QAT state1 length in bytes */
+	uint32_t qshqi_state2_len;	/* QAT state2 length in bytes */
+};
+
+struct qat_sym_hash_def {
+	const struct qat_sym_hash_alg_info *qshd_alg;
+	const struct qat_sym_hash_qat_info *qshd_qat;
+};
+
+#define QAT_SYM_REQ_PARAMS_SIZE_MAX			(24 + 32)
+/* Reserve enough space for cipher and authentication request params */
+/* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */
+
+#define QAT_SYM_REQ_PARAMS_SIZE_PADDED			\
+		roundup(QAT_SYM_REQ_PARAMS_SIZE_MAX, QAT_OPTIMAL_ALIGN)
+/* Pad out to 64-byte multiple to ensure optimal alignment of next field */
+
+#define QAT_SYM_KEY_TLS_PREFIX_SIZE			(128)
+/* Hash Prefix size in bytes for TLS (128 = MAX = SHA2 (384, 512)*/
+
+#define QAT_SYM_KEY_MAX_HASH_STATE_BUFFER		\
+		(QAT_SYM_KEY_TLS_PREFIX_SIZE * 2)
+/* hash state prefix buffer structure that holds the maximum sized secret */
+
+#define QAT_SYM_HASH_BUFFER_LEN			QAT_HASH_SHA512_STATE_SIZE
+/* Buffer length to hold 16 byte MD5 key and 20 byte SHA1 key */
+
+#define QAT_GCM_AAD_SIZE_MAX		240
+/* Maximum AAD size */
+
+#define	QAT_AES_GCM_AAD_ALIGN		16
+
+struct qat_sym_bulk_cookie {
+	uint8_t qsbc_req_params_buf[QAT_SYM_REQ_PARAMS_SIZE_PADDED];
+	/* memory block reserved for request params
+	 * NOTE: Field must be correctly aligned in memory for access by QAT
+	 * engine */
+	struct qat_crypto *qsbc_crypto;
+	struct qat_session *qsbc_session;
+	/* Session context */
+	void *qsbc_cb_tag;
+	/* correlator supplied by the client */
+	uint8_t qsbc_msg[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
+	/* QAT request message */
+} __aligned(QAT_OPTIMAL_ALIGN);
+
+/* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */
+#define HASH_CONTENT_DESC_SIZE		176
+#define CIPHER_CONTENT_DESC_SIZE	64
+
+#define CONTENT_DESC_MAX_SIZE	roundup(				\
+		HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE,	\
+		QAT_OPTIMAL_ALIGN)
+
+struct qat_sym_cookie {
+	union qat_sym_cookie_u {
+		/* should be 64byte aligned */
+		struct qat_sym_bulk_cookie qsc_bulk_cookie;
+						/* symmetric bulk cookie */
+#ifdef notyet
+		struct qat_sym_key_cookie qsc_key_cookie;
+						/* symmetric key cookie */
+		struct qat_sym_nrbg_cookie qsc_nrbg_cookie;
+						/* symmetric NRBG cookie */
+#endif
+	} u;
+
+	/* should be 64-byte aligned */
+	struct buffer_list_desc qsc_buf_list;
+	struct flat_buffer_desc qsc_flat_bufs[QAT_MAXSEG]; /* should be here */
+
+	bus_dmamap_t qsc_self_dmamap;	/* self DMA mapping and
+					   end of DMA region */
+	bus_dma_tag_t qsc_self_dma_tag;
+
+	uint8_t qsc_iv_buf[EALG_MAX_BLOCK_LEN];
+	uint8_t qsc_auth_res[QAT_SYM_HASH_BUFFER_LEN];
+	uint8_t qsc_gcm_aad[QAT_GCM_AAD_SIZE_MAX];
+	uint8_t qsc_content_desc[CONTENT_DESC_MAX_SIZE];
+
+	bus_dmamap_t qsc_buf_dmamap;	/* qsc_flat_bufs DMA mapping */
+	bus_dma_tag_t qsc_buf_dma_tag;
+	void *qsc_buf;
+
+	bus_addr_t qsc_bulk_req_params_buf_paddr;
+	bus_addr_t qsc_buffer_list_desc_paddr;
+	bus_addr_t qsc_iv_buf_paddr;
+	bus_addr_t qsc_auth_res_paddr;
+	bus_addr_t qsc_gcm_aad_paddr;
+	bus_addr_t qsc_content_desc_paddr;
+};
+
+CTASSERT(offsetof(struct qat_sym_cookie,
+    u.qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0);
+CTASSERT(offsetof(struct qat_sym_cookie, qsc_buf_list) % QAT_OPTIMAL_ALIGN == 0);
+
+#define MAX_CIPHER_SETUP_BLK_SZ						\
+		(sizeof(struct hw_cipher_config) +			\
+		2 * HW_KASUMI_KEY_SZ + 2 * HW_KASUMI_BLK_SZ)
+#define MAX_HASH_SETUP_BLK_SZ	sizeof(union hw_auth_algo_blk)
+
+struct qat_crypto_desc {
+	uint8_t qcd_content_desc[CONTENT_DESC_MAX_SIZE]; /* must be first */
+	/* using only for qat 1.5 */
+	uint8_t qcd_hash_state_prefix_buf[QAT_GCM_AAD_SIZE_MAX];
+
+	bus_addr_t qcd_desc_paddr;
+	bus_addr_t qcd_hash_state_paddr;
+
+	enum fw_slice qcd_slices[MAX_FW_SLICE];
+	enum fw_la_cmd_id qcd_cmd_id;
+	enum hw_cipher_dir qcd_cipher_dir;
+
+	/* content desc info */
+	uint8_t qcd_hdr_sz;		/* in quad words */
+	uint8_t qcd_hw_blk_sz;		/* in quad words */
+	uint32_t qcd_cipher_offset;
+	uint32_t qcd_auth_offset;
+	/* hash info */
+	uint8_t qcd_state_storage_sz;	/* in quad words */
+	uint32_t qcd_gcm_aad_sz_offset1;
+	uint32_t qcd_gcm_aad_sz_offset2;
+	/* cipher info */
+	uint16_t qcd_cipher_blk_sz;	/* in bytes */
+	uint16_t qcd_auth_sz;		/* in bytes */
+
+	uint8_t qcd_req_cache[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
+} __aligned(QAT_OPTIMAL_ALIGN);
+
+/* should be aligned to 64bytes */
+struct qat_session {
+	struct qat_crypto_desc *qs_dec_desc;	/* should be at top of struct*/
+	/* decrypt or auth then decrypt or auth */
+
+	struct qat_crypto_desc *qs_enc_desc;
+	/* encrypt or encrypt then auth */
+
+	struct qat_dmamem qs_desc_mem;
+
+	enum hw_cipher_algo qs_cipher_algo;
+	enum hw_cipher_mode qs_cipher_mode;
+	enum hw_auth_algo qs_auth_algo;
+	enum hw_auth_mode qs_auth_mode;
+
+	const uint8_t *qs_cipher_key;
+	int qs_cipher_klen;
+	const uint8_t *qs_auth_key;
+	int qs_auth_klen;
+	int qs_auth_mlen;
+
+	uint32_t qs_status;
+#define QAT_SESSION_STATUS_ACTIVE	(1 << 0)
+#define QAT_SESSION_STATUS_FREEING	(1 << 1)
+	uint32_t qs_inflight;
+	int qs_aad_length;
+	bool qs_need_wakeup;
+
+	struct mtx qs_session_mtx;
+};
+
+struct qat_crypto_bank {
+	uint16_t qcb_bank;
+
+	struct qat_ring *qcb_sym_tx;
+	struct qat_ring *qcb_sym_rx;
+
+	struct qat_dmamem qcb_symck_dmamems[QAT_NSYMCOOKIE];
+	struct qat_sym_cookie *qcb_symck_free[QAT_NSYMCOOKIE];
+	uint32_t qcb_symck_free_count;
+
+	struct mtx qcb_bank_mtx;
+
+	char qcb_ring_names[2][QAT_RING_NAME_SIZE];	/* sym tx,rx */
+};
+
+struct qat_crypto {
+	struct qat_softc *qcy_sc;
+	uint32_t qcy_bank_mask;
+	uint16_t qcy_num_banks;
+
+	int32_t qcy_cid;		/* OpenCrypto driver ID */
+
+	struct qat_crypto_bank *qcy_banks; /* array of qat_crypto_bank */
+
+	uint32_t qcy_session_free_count;
+
+	struct mtx qcy_crypto_mtx;
+};
+
+struct qat_hw {
+	int8_t qhw_sram_bar_id;
+	int8_t qhw_misc_bar_id;
+	int8_t qhw_etr_bar_id;
+
+	bus_size_t qhw_cap_global_offset;
+	bus_size_t qhw_ae_offset;
+	bus_size_t qhw_ae_local_offset;
+	bus_size_t qhw_etr_bundle_size;
+
+	/* crypto processing callbacks */
+	size_t qhw_crypto_opaque_offset;
+	void (*qhw_crypto_setup_req_params)(struct qat_crypto_bank *,
+	    struct qat_session *, struct qat_crypto_desc const *,
+	    struct qat_sym_cookie *, struct cryptop *);
+	void (*qhw_crypto_setup_desc)(struct qat_crypto *, struct qat_session *,
+	    struct qat_crypto_desc *);
+
+	uint8_t qhw_num_banks;			/* max number of banks */
+	uint8_t qhw_num_ap_banks;		/* max number of AutoPush banks */
+	uint8_t qhw_num_rings_per_bank;		/* rings per bank */
+	uint8_t qhw_num_accel;			/* max number of accelerators */
+	uint8_t qhw_num_engines;		/* max number of accelerator engines */
+	uint8_t qhw_tx_rx_gap;
+	uint32_t qhw_tx_rings_mask;
+	uint32_t qhw_clock_per_sec;
+	bool qhw_fw_auth;
+	uint32_t qhw_fw_req_size;
+	uint32_t qhw_fw_resp_size;
+
+	uint8_t qhw_ring_sym_tx;
+	uint8_t qhw_ring_sym_rx;
+	uint8_t qhw_ring_asym_tx;
+	uint8_t qhw_ring_asym_rx;
+
+	/* MSIx */
+	uint32_t qhw_msix_ae_vec_gap;	/* gap to ae vec from bank */
+
+	const char *qhw_mof_fwname;
+	const char *qhw_mmp_fwname;
+
+	uint32_t qhw_prod_type;		/* cpu type */
+
+	/* setup callbacks */
+	uint32_t (*qhw_get_accel_mask)(struct qat_softc *);
+	uint32_t (*qhw_get_ae_mask)(struct qat_softc *);
+	enum qat_sku (*qhw_get_sku)(struct qat_softc *);
+	uint32_t (*qhw_get_accel_cap)(struct qat_softc *);
+	const char *(*qhw_get_fw_uof_name)(struct qat_softc *);
+	void (*qhw_enable_intr)(struct qat_softc *);
+	void (*qhw_init_etr_intr)(struct qat_softc *, int);
+	int (*qhw_init_admin_comms)(struct qat_softc *);
+	int (*qhw_send_admin_init)(struct qat_softc *);
+	int (*qhw_init_arb)(struct qat_softc *);
+	void (*qhw_get_arb_mapping)(struct qat_softc *, const uint32_t **);
+	void (*qhw_enable_error_correction)(struct qat_softc *);
+	int (*qhw_check_uncorrectable_error)(struct qat_softc *);
+	void (*qhw_print_err_registers)(struct qat_softc *);
+	void (*qhw_disable_error_interrupts)(struct qat_softc *);
+	int (*qhw_check_slice_hang)(struct qat_softc *);
+	int (*qhw_set_ssm_wdtimer)(struct qat_softc *);
+};
+
+
+/* sc_flags */
+#define QAT_FLAG_ESRAM_ENABLE_AUTO_INIT	(1 << 0)
+#define QAT_FLAG_SHRAM_WAIT_READY	(1 << 1)
+
+/* sc_accel_cap */
+#define QAT_ACCEL_CAP_CRYPTO_SYMMETRIC	(1 << 0)
+#define QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC	(1 << 1)
+#define QAT_ACCEL_CAP_CIPHER		(1 << 2)
+#define QAT_ACCEL_CAP_AUTHENTICATION	(1 << 3)
+#define QAT_ACCEL_CAP_REGEX		(1 << 4)
+#define QAT_ACCEL_CAP_COMPRESSION	(1 << 5)
+#define QAT_ACCEL_CAP_LZS_COMPRESSION	(1 << 6)
+#define QAT_ACCEL_CAP_RANDOM_NUMBER	(1 << 7)
+#define QAT_ACCEL_CAP_ZUC		(1 << 8)
+#define QAT_ACCEL_CAP_SHA3		(1 << 9)
+#define QAT_ACCEL_CAP_KPT		(1 << 10)
+
+#define QAT_ACCEL_CAP_BITS	\
+	"\177\020"	\
+	"b\x0a"		"KPT\0" \
+	"b\x09"		"SHA3\0" \
+	"b\x08"		"ZUC\0" \
+	"b\x07"		"RANDOM_NUMBER\0" \
+	"b\x06"		"LZS_COMPRESSION\0" \
+	"b\x05"		"COMPRESSION\0" \
+	"b\x04"		"REGEX\0" \
+	"b\x03"		"AUTHENTICATION\0" \
+	"b\x02"		"CIPHER\0" \
+	"b\x01"		"CRYPTO_ASYMMETRIC\0" \
+	"b\x00"		"CRYPTO_SYMMETRIC\0"
+
+#define QAT_HI_PRIO_RING_WEIGHT		0xfc
+#define QAT_LO_PRIO_RING_WEIGHT		0xfe
+#define QAT_DEFAULT_RING_WEIGHT		0xff
+#define QAT_DEFAULT_PVL			0
+
+struct firmware;
+struct resource;
+
+struct qat_softc {
+	device_t sc_dev;
+
+	struct resource *sc_res[MAX_BARS];
+	int sc_rid[MAX_BARS];
+	bus_space_tag_t sc_csrt[MAX_BARS];
+	bus_space_handle_t sc_csrh[MAX_BARS];
+
+	uint32_t sc_ae_num;
+	uint32_t sc_ae_mask;
+
+	struct qat_crypto sc_crypto;		/* crypto services */
+
+	struct qat_hw sc_hw;
+
+	uint8_t sc_rev;
+	enum qat_sku sc_sku;
+	uint32_t sc_flags;
+
+	uint32_t sc_accel_num;
+	uint32_t sc_accel_mask;
+	uint32_t sc_accel_cap;
+
+	struct qat_admin_rings sc_admin_rings;	/* use only for qat 1.5 */
+	struct qat_admin_comms sc_admin_comms;	/* use only for qat 1.7 */
+
+	/* ETR */
+	struct qat_bank *sc_etr_banks;		/* array of etr banks */
+	struct qat_ap_bank *sc_etr_ap_banks;	/* array of etr auto push banks */
+
+	/* AE */
+	struct qat_ae sc_ae[MAX_NUM_AE];
+
+	/* Interrupt */
+	struct resource *sc_ih;			/* ae cluster ih */
+	void *sc_ih_cookie;			/* ae cluster ih cookie */
+
+	/* Counters */
+	counter_u64_t sc_gcm_aad_restarts;
+	counter_u64_t sc_gcm_aad_updates;
+	counter_u64_t sc_ring_full_restarts;
+
+	/* Firmware */
+	void *sc_fw_mof;			/* mof data */
+	size_t sc_fw_mof_size;			/* mof size */
+	struct qat_mof sc_mof;			/* mof sections */
+
+	const char *sc_fw_uof_name;		/* uof/suof name in mof */
+
+	void *sc_fw_uof;			/* uof head */
+	size_t sc_fw_uof_size;			/* uof size */
+	struct qat_aefw_uof sc_aefw_uof;	/* UOF_OBJS in uof */
+
+	void *sc_fw_suof;			/* suof head */
+	size_t sc_fw_suof_size;			/* suof size */
+	struct qat_aefw_suof sc_aefw_suof;	/* suof context */
+
+	void *sc_fw_mmp;			/* mmp data */
+	size_t sc_fw_mmp_size;			/* mmp size */
+};
+
+static inline void
+qat_bar_write_4(struct qat_softc *sc, int baroff, bus_size_t offset,
+    uint32_t value)
+{
+
+	MPASS(baroff >= 0 && baroff < MAX_BARS);
+
+	bus_space_write_4(sc->sc_csrt[baroff],
+	    sc->sc_csrh[baroff], offset, value);
+}
+
+static inline uint32_t
+qat_bar_read_4(struct qat_softc *sc, int baroff, bus_size_t offset)
+{
+
+	MPASS(baroff >= 0 && baroff < MAX_BARS);
+
+	return bus_space_read_4(sc->sc_csrt[baroff],
+	    sc->sc_csrh[baroff], offset);
+}
+
+static inline void
+qat_misc_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
+{
+
+	qat_bar_write_4(sc, sc->sc_hw.qhw_misc_bar_id, offset, value);
+}
+
+static inline uint32_t
+qat_misc_read_4(struct qat_softc *sc, bus_size_t offset)
+{
+
+	return qat_bar_read_4(sc, sc->sc_hw.qhw_misc_bar_id, offset);
+}
+
+static inline void
+qat_misc_read_write_or_4(struct qat_softc *sc, bus_size_t offset,
+    uint32_t value)
+{
+	uint32_t reg;
+
+	reg = qat_misc_read_4(sc, offset);
+	reg |= value;
+	qat_misc_write_4(sc, offset, reg);
+}
+
+static inline void
+qat_misc_read_write_and_4(struct qat_softc *sc, bus_size_t offset,
+    uint32_t mask)
+{
+	uint32_t reg;
+
+	reg = qat_misc_read_4(sc, offset);
+	reg &= mask;
+	qat_misc_write_4(sc, offset, reg);
+}
+
+static inline void
+qat_etr_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
+{
+
+	qat_bar_write_4(sc, sc->sc_hw.qhw_etr_bar_id, offset, value);
+}
+
+static inline uint32_t
+qat_etr_read_4(struct qat_softc *sc, bus_size_t offset)
+{
+
+	return qat_bar_read_4(sc, sc->sc_hw.qhw_etr_bar_id, offset);
+}
+
+static inline void
+qat_ae_local_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
+	uint32_t value)
+{
+
+	offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) |
+	    (offset & AE_LOCAL_CSR_MASK);
+
+	qat_misc_write_4(sc, sc->sc_hw.qhw_ae_local_offset + offset,
+	    value);
+}
+
+static inline uint32_t
+qat_ae_local_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset)
+{
+
+	offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) |
+	    (offset & AE_LOCAL_CSR_MASK);
+
+	return qat_misc_read_4(sc, sc->sc_hw.qhw_ae_local_offset + offset);
+}
+
+static inline void
+qat_ae_xfer_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
+	uint32_t value)
+{
+	offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_XFER_AE_MASK) |
+	    __SHIFTIN(offset, AE_XFER_CSR_MASK);
+
+	qat_misc_write_4(sc, sc->sc_hw.qhw_ae_offset + offset, value);
+}
+
+static inline void
+qat_cap_global_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
+{
+
+	qat_misc_write_4(sc, sc->sc_hw.qhw_cap_global_offset + offset, value);
+}
+
+static inline uint32_t
+qat_cap_global_read_4(struct qat_softc *sc, bus_size_t offset)
+{
+
+	return qat_misc_read_4(sc, sc->sc_hw.qhw_cap_global_offset + offset);
+}
+
+
+static inline void
+qat_etr_bank_write_4(struct qat_softc *sc, int bank,
+	bus_size_t offset, uint32_t value)
+{
+
+	qat_etr_write_4(sc, sc->sc_hw.qhw_etr_bundle_size * bank + offset,
+	    value);
+}
+
+static inline uint32_t
+qat_etr_bank_read_4(struct qat_softc *sc, int bank,
+	bus_size_t offset)
+{
+
+	return qat_etr_read_4(sc,
+	    sc->sc_hw.qhw_etr_bundle_size * bank + offset);
+}
+
+static inline void
+qat_etr_ap_bank_write_4(struct qat_softc *sc, int ap_bank,
+	bus_size_t offset, uint32_t value)
+{
+
+	qat_etr_write_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset, value);
+}
+
+static inline uint32_t
+qat_etr_ap_bank_read_4(struct qat_softc *sc, int ap_bank,
+	bus_size_t offset)
+{
+
+	return qat_etr_read_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset);
+}
+
+
+static inline void
+qat_etr_bank_ring_write_4(struct qat_softc *sc, int bank, int ring,
+	bus_size_t offset, uint32_t value)
+{
+
+	qat_etr_bank_write_4(sc, bank, (ring << 2) + offset, value);
+}
+
+static inline uint32_t
+qat_etr_bank_ring_read_4(struct qat_softc *sc, int bank, int ring,
+	bus_size_t offset)
+{
+
+	return qat_etr_bank_read_4(sc, bank, (ring << 2) * offset);
+}
+
+static inline void
+qat_etr_bank_ring_base_write_8(struct qat_softc *sc, int bank, int ring,
+	uint64_t value)
+{
+	uint32_t lo, hi;
+
+	lo = (uint32_t)(value & 0xffffffff);
+	hi = (uint32_t)((value & 0xffffffff00000000ULL) >> 32);
+	qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_LBASE, lo);
+	qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_UBASE, hi);
+}
+
+static inline void
+qat_arb_ringsrvarben_write_4(struct qat_softc *sc, int index, uint32_t value)
+{
+
+	qat_etr_write_4(sc, ARB_RINGSRVARBEN_OFFSET +
+	    (ARB_REG_SLOT * index), value);
+}
+
+static inline void
+qat_arb_sarconfig_write_4(struct qat_softc *sc, int index, uint32_t value)
+{
+
+	qat_etr_write_4(sc, ARB_OFFSET +
+	    (ARB_REG_SIZE * index), value);
+}
+
+static inline void
+qat_arb_wrk_2_ser_map_write_4(struct qat_softc *sc, int index, uint32_t value)
+{
+
+	qat_etr_write_4(sc, ARB_OFFSET + ARB_WRK_2_SER_MAP_OFFSET +
+	    (ARB_REG_SIZE * index), value);
+}
+
+void *		qat_alloc_mem(size_t);
+void		qat_free_mem(void *);
+void		qat_free_dmamem(struct qat_softc *, struct qat_dmamem *);
+int		qat_alloc_dmamem(struct qat_softc *, struct qat_dmamem *, int,
+		    bus_size_t, bus_size_t);
+
+int		qat_etr_setup_ring(struct qat_softc *, int, uint32_t, uint32_t,
+		    uint32_t, qat_cb_t, void *, const char *,
+		    struct qat_ring **);
+int		qat_etr_put_msg(struct qat_softc *, struct qat_ring *,
+		    uint32_t *);
+
+void		qat_memcpy_htobe64(void *, const void *, size_t);
+void		qat_memcpy_htobe32(void *, const void *, size_t);
+void		qat_memcpy_htobe(void *, const void *, size_t, uint32_t);
+void		qat_crypto_gmac_precompute(const struct qat_crypto_desc *,
+		    const uint8_t *key, int klen,
+		    const struct qat_sym_hash_def *, uint8_t *);
+void		qat_crypto_hmac_precompute(const struct qat_crypto_desc *,
+		    const uint8_t *, int, const struct qat_sym_hash_def *,
+		    uint8_t *, uint8_t *);
+uint16_t	qat_crypto_load_cipher_session(const struct qat_crypto_desc *,
+		    const struct qat_session *);
+uint16_t	qat_crypto_load_auth_session(const struct qat_crypto_desc *,
+		    const struct qat_session *,
+		    struct qat_sym_hash_def const **);
+
+#endif
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index b0feddbad301..58cdf5071436 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -300,6 +300,7 @@ SUBDIR=	\
 	pty  \
 	puc \
 	pwm \
+	${_qat} \
 	${_qlxge} \
 	${_qlxgb} \
 	${_qlxgbe} \
@@ -630,6 +631,7 @@ _ntb=		ntb
 _ocs_fc=	ocs_fc
 _ossl=		ossl
 _pccard=	pccard
+_qat=		qat
 .if ${MK_OFED} != "no" || defined(ALL_MODULES)
 _rdma=		rdma
 .endif
diff --git a/sys/modules/qat/Makefile b/sys/modules/qat/Makefile
new file mode 100644
index 000000000000..57301b5b78e3
--- /dev/null
+++ b/sys/modules/qat/Makefile
@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+.PATH:	${SRCTOP}/sys/dev/qat
+
+KMOD=	qat
+
+SRCS=	qat.c		\
+	qat_ae.c	\
+	qat_c2xxx.c	\
+	qat_c3xxx.c	\
+	qat_c62x.c	\
+	qat_d15xx.c	\
+	qat_dh895xcc.c	\
+	qat_hw15.c	\
+	qat_hw17.c
+
+SRCS+=	bus_if.h cryptodev_if.h device_if.h pci_if.h
+
+.include <bsd.kmod.mk>