aboutsummaryrefslogtreecommitdiff
path: root/sys/contrib/openzfs/module/icp
diff options
context:
space:
mode:
authorMatt Macy <mmacy@FreeBSD.org>2021-01-07 23:27:17 +0000
committerMatt Macy <mmacy@FreeBSD.org>2021-01-08 00:55:59 +0000
commit7877fdebeeb35fad1cbbafce22598b1bdf97c786 (patch)
tree10ccc0bab059d6f48a221045b92416fc347fe784 /sys/contrib/openzfs/module/icp
parent84089de83e79a0f748c6e22b1aacb59156e153d2 (diff)
downloadsrc-7877fdebeeb35fad1cbbafce22598b1bdf97c786.tar.gz
src-7877fdebeeb35fad1cbbafce22598b1bdf97c786.zip
OpenZFS merge main-gf11b09
- add dRAID support - fix duplicate close handling - fix memory leak in prefetch - fix problem with SIMD benchmarking on FreeBSD boot ...
Diffstat (limited to 'sys/contrib/openzfs/module/icp')
-rw-r--r--sys/contrib/openzfs/module/icp/algs/modes/gcm.c54
-rw-r--r--sys/contrib/openzfs/module/icp/algs/modes/modes.c8
-rw-r--r--sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S24
-rw-r--r--sys/contrib/openzfs/module/icp/core/kcf_sched.c4
-rw-r--r--sys/contrib/openzfs/module/icp/include/modes/modes.h8
-rw-r--r--sys/contrib/openzfs/module/icp/io/aes.c18
6 files changed, 100 insertions, 16 deletions
diff --git a/sys/contrib/openzfs/module/icp/algs/modes/gcm.c b/sys/contrib/openzfs/module/icp/algs/modes/gcm.c
index 5553c55e11cd..23686c59e8ce 100644
--- a/sys/contrib/openzfs/module/icp/algs/modes/gcm.c
+++ b/sys/contrib/openzfs/module/icp/algs/modes/gcm.c
@@ -59,10 +59,12 @@ boolean_t gcm_avx_can_use_movbe = B_FALSE;
static boolean_t gcm_use_avx = B_FALSE;
#define GCM_IMPL_USE_AVX (*(volatile boolean_t *)&gcm_use_avx)
+extern boolean_t atomic_toggle_boolean_nv(volatile boolean_t *);
+
static inline boolean_t gcm_avx_will_work(void);
static inline void gcm_set_avx(boolean_t);
static inline boolean_t gcm_toggle_avx(void);
-extern boolean_t atomic_toggle_boolean_nv(volatile boolean_t *);
+static inline size_t gcm_simd_get_htab_size(boolean_t);
static int gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *, char *, size_t,
crypto_data_t *, size_t);
@@ -629,6 +631,21 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
(volatile boolean_t *)&gcm_avx_can_use_movbe);
}
}
+ /* Allocate Htab memory as needed. */
+ if (gcm_ctx->gcm_use_avx == B_TRUE) {
+ size_t htab_len = gcm_simd_get_htab_size(gcm_ctx->gcm_use_avx);
+
+ if (htab_len == 0) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ gcm_ctx->gcm_htab_len = htab_len;
+ gcm_ctx->gcm_Htable =
+ (uint64_t *)kmem_alloc(htab_len, gcm_ctx->gcm_kmflag);
+
+ if (gcm_ctx->gcm_Htable == NULL) {
+ return (CRYPTO_HOST_MEMORY);
+ }
+ }
/* Avx and non avx context initialization differs from here on. */
if (gcm_ctx->gcm_use_avx == B_FALSE) {
#endif /* ifdef CAN_USE_GCM_ASM */
@@ -689,6 +706,22 @@ gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
if (ks->ops->needs_byteswap == B_TRUE) {
gcm_ctx->gcm_use_avx = B_FALSE;
}
+ /* Allocate Htab memory as needed. */
+ if (gcm_ctx->gcm_use_avx == B_TRUE) {
+ size_t htab_len = gcm_simd_get_htab_size(gcm_ctx->gcm_use_avx);
+
+ if (htab_len == 0) {
+ return (CRYPTO_MECHANISM_PARAM_INVALID);
+ }
+ gcm_ctx->gcm_htab_len = htab_len;
+ gcm_ctx->gcm_Htable =
+ (uint64_t *)kmem_alloc(htab_len, gcm_ctx->gcm_kmflag);
+
+ if (gcm_ctx->gcm_Htable == NULL) {
+ return (CRYPTO_HOST_MEMORY);
+ }
+ }
+
/* Avx and non avx context initialization differs from here on. */
if (gcm_ctx->gcm_use_avx == B_FALSE) {
#endif /* ifdef CAN_USE_GCM_ASM */
@@ -1018,7 +1051,7 @@ MODULE_PARM_DESC(icp_gcm_impl, "Select gcm implementation.");
/* Clear the FPU registers since they hold sensitive internal state. */
#define clear_fpu_regs() clear_fpu_regs_avx()
#define GHASH_AVX(ctx, in, len) \
- gcm_ghash_avx((ctx)->gcm_ghash, (const uint64_t (*)[2])(ctx)->gcm_Htable, \
+ gcm_ghash_avx((ctx)->gcm_ghash, (const uint64_t *)(ctx)->gcm_Htable, \
in, len)
#define gcm_incr_counter_block(ctx) gcm_incr_counter_block_by(ctx, 1)
@@ -1036,8 +1069,8 @@ extern void gcm_xor_avx(const uint8_t *src, uint8_t *dst);
extern void aes_encrypt_intel(const uint32_t rk[], int nr,
const uint32_t pt[4], uint32_t ct[4]);
-extern void gcm_init_htab_avx(uint64_t Htable[16][2], const uint64_t H[2]);
-extern void gcm_ghash_avx(uint64_t ghash[2], const uint64_t Htable[16][2],
+extern void gcm_init_htab_avx(uint64_t *Htable, const uint64_t H[2]);
+extern void gcm_ghash_avx(uint64_t ghash[2], const uint64_t *Htable,
const uint8_t *in, size_t len);
extern size_t aesni_gcm_encrypt(const uint8_t *, uint8_t *, size_t,
@@ -1073,6 +1106,18 @@ gcm_toggle_avx(void)
}
}
+static inline size_t
+gcm_simd_get_htab_size(boolean_t simd_mode)
+{
+ switch (simd_mode) {
+ case B_TRUE:
+ return (2 * 6 * 2 * sizeof (uint64_t));
+
+ default:
+ return (0);
+ }
+}
+
/*
* Clear sensitive data in the context.
*
@@ -1088,7 +1133,6 @@ gcm_clear_ctx(gcm_ctx_t *ctx)
{
bzero(ctx->gcm_remainder, sizeof (ctx->gcm_remainder));
bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
- bzero(ctx->gcm_Htable, sizeof (ctx->gcm_Htable));
bzero(ctx->gcm_J0, sizeof (ctx->gcm_J0));
bzero(ctx->gcm_tmp, sizeof (ctx->gcm_tmp));
}
diff --git a/sys/contrib/openzfs/module/icp/algs/modes/modes.c b/sys/contrib/openzfs/module/icp/algs/modes/modes.c
index f07876a478e2..faae9722bd04 100644
--- a/sys/contrib/openzfs/module/icp/algs/modes/modes.c
+++ b/sys/contrib/openzfs/module/icp/algs/modes/modes.c
@@ -152,6 +152,14 @@ crypto_free_mode_ctx(void *ctx)
vmem_free(((gcm_ctx_t *)ctx)->gcm_pt_buf,
((gcm_ctx_t *)ctx)->gcm_pt_buf_len);
+#ifdef CAN_USE_GCM_ASM
+ if (((gcm_ctx_t *)ctx)->gcm_Htable != NULL) {
+ gcm_ctx_t *gcm_ctx = (gcm_ctx_t *)ctx;
+ bzero(gcm_ctx->gcm_Htable, gcm_ctx->gcm_htab_len);
+ kmem_free(gcm_ctx->gcm_Htable, gcm_ctx->gcm_htab_len);
+ }
+#endif
+
kmem_free(ctx, sizeof (gcm_ctx_t));
}
}
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
index ed9f660fce5b..dc71ae2c1c89 100644
--- a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
@@ -55,6 +55,7 @@
.type _aesni_ctr32_ghash_6x,@function
.align 32
_aesni_ctr32_ghash_6x:
+.cfi_startproc
vmovdqu 32(%r11),%xmm2
subq $6,%rdx
vpxor %xmm4,%xmm4,%xmm4
@@ -363,12 +364,14 @@ _aesni_ctr32_ghash_6x:
vpxor %xmm4,%xmm8,%xmm8
.byte 0xf3,0xc3
+.cfi_endproc
.size _aesni_ctr32_ghash_6x,.-_aesni_ctr32_ghash_6x
#endif /* ifdef HAVE_MOVBE */
.type _aesni_ctr32_ghash_no_movbe_6x,@function
.align 32
_aesni_ctr32_ghash_no_movbe_6x:
+.cfi_startproc
vmovdqu 32(%r11),%xmm2
subq $6,%rdx
vpxor %xmm4,%xmm4,%xmm4
@@ -689,6 +692,7 @@ _aesni_ctr32_ghash_no_movbe_6x:
vpxor %xmm4,%xmm8,%xmm8
.byte 0xf3,0xc3
+.cfi_endproc
.size _aesni_ctr32_ghash_no_movbe_6x,.-_aesni_ctr32_ghash_no_movbe_6x
.globl aesni_gcm_decrypt
@@ -714,6 +718,8 @@ aesni_gcm_decrypt:
.cfi_offset %r14,-48
pushq %r15
.cfi_offset %r15,-56
+ pushq %r9
+.cfi_offset %r9,-64
vzeroupper
vmovdqu (%r8),%xmm1
@@ -726,7 +732,8 @@ aesni_gcm_decrypt:
andq $-128,%rsp
vmovdqu (%r11),%xmm0
leaq 128(%rcx),%rcx
- leaq 32+32(%r9),%r9
+ movq 32(%r9),%r9
+ leaq 32(%r9),%r9
movl 504-128(%rcx),%ebp // ICP has a larger offset for rounds.
vpshufb %xmm0,%xmm8,%xmm8
@@ -782,7 +789,9 @@ aesni_gcm_decrypt:
vmovups %xmm14,-16(%rsi)
vpshufb (%r11),%xmm8,%xmm8
- vmovdqu %xmm8,-64(%r9)
+ movq -56(%rax),%r9
+.cfi_restore %r9
+ vmovdqu %xmm8,(%r9)
vzeroupper
movq -48(%rax),%r15
@@ -807,6 +816,7 @@ aesni_gcm_decrypt:
.type _aesni_ctr32_6x,@function
.align 32
_aesni_ctr32_6x:
+.cfi_startproc
vmovdqu 0-128(%rcx),%xmm4
vmovdqu 32(%r11),%xmm2
leaq -2(%rbp),%r13 // ICP uses 10,12,14 not 9,11,13 for rounds.
@@ -893,6 +903,7 @@ _aesni_ctr32_6x:
vpshufb %xmm0,%xmm1,%xmm1
vpxor %xmm4,%xmm14,%xmm14
jmp .Loop_ctr32
+.cfi_endproc
.size _aesni_ctr32_6x,.-_aesni_ctr32_6x
.globl aesni_gcm_encrypt
@@ -918,6 +929,8 @@ aesni_gcm_encrypt:
.cfi_offset %r14,-48
pushq %r15
.cfi_offset %r15,-56
+ pushq %r9
+.cfi_offset %r9,-64
vzeroupper
vmovdqu (%r8),%xmm1
@@ -960,7 +973,8 @@ aesni_gcm_encrypt:
call _aesni_ctr32_6x
vmovdqu (%r9),%xmm8
- leaq 32+32(%r9),%r9
+ movq 32(%r9),%r9
+ leaq 32(%r9),%r9
subq $12,%rdx
movq $192,%r10
vpshufb %xmm0,%xmm8,%xmm8
@@ -1151,7 +1165,9 @@ aesni_gcm_encrypt:
vpxor %xmm7,%xmm2,%xmm2
vpxor %xmm2,%xmm8,%xmm8
vpshufb (%r11),%xmm8,%xmm8
- vmovdqu %xmm8,-64(%r9)
+ movq -56(%rax),%r9
+.cfi_restore %r9
+ vmovdqu %xmm8,(%r9)
vzeroupper
movq -48(%rax),%r15
diff --git a/sys/contrib/openzfs/module/icp/core/kcf_sched.c b/sys/contrib/openzfs/module/icp/core/kcf_sched.c
index 40d50553d67e..81fd15f8ea26 100644
--- a/sys/contrib/openzfs/module/icp/core/kcf_sched.c
+++ b/sys/contrib/openzfs/module/icp/core/kcf_sched.c
@@ -1308,9 +1308,7 @@ kcf_reqid_insert(kcf_areq_node_t *areq)
kcf_areq_node_t *headp;
kcf_reqid_table_t *rt;
- kpreempt_disable();
- rt = kcf_reqid_table[CPU_SEQID & REQID_TABLE_MASK];
- kpreempt_enable();
+ rt = kcf_reqid_table[CPU_SEQID_UNSTABLE & REQID_TABLE_MASK];
mutex_enter(&rt->rt_lock);
diff --git a/sys/contrib/openzfs/module/icp/include/modes/modes.h b/sys/contrib/openzfs/module/icp/include/modes/modes.h
index 57a211ccf1bf..ab71197542eb 100644
--- a/sys/contrib/openzfs/module/icp/include/modes/modes.h
+++ b/sys/contrib/openzfs/module/icp/include/modes/modes.h
@@ -219,14 +219,14 @@ typedef struct gcm_ctx {
size_t gcm_pt_buf_len;
uint32_t gcm_tmp[4];
/*
- * The relative positions of gcm_ghash, gcm_H and pre-computed
- * gcm_Htable are hard coded in aesni-gcm-x86_64.S and ghash-x86_64.S,
- * so please don't change (or adjust accordingly).
+ * The offset of gcm_Htable relative to gcm_ghash, (32), is hard coded
+ * in aesni-gcm-x86_64.S, so please don't change (or adjust there).
*/
uint64_t gcm_ghash[2];
uint64_t gcm_H[2];
#ifdef CAN_USE_GCM_ASM
- uint64_t gcm_Htable[12][2];
+ uint64_t *gcm_Htable;
+ size_t gcm_htab_len;
#endif
uint64_t gcm_J0[2];
uint64_t gcm_len_a_len_c[2];
diff --git a/sys/contrib/openzfs/module/icp/io/aes.c b/sys/contrib/openzfs/module/icp/io/aes.c
index 96fb6bb1af30..e540af4473f7 100644
--- a/sys/contrib/openzfs/module/icp/io/aes.c
+++ b/sys/contrib/openzfs/module/icp/io/aes.c
@@ -1051,6 +1051,16 @@ out:
bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
}
+#ifdef CAN_USE_GCM_ASM
+ if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE) &&
+ ((gcm_ctx_t *)&aes_ctx)->gcm_Htable != NULL) {
+
+ gcm_ctx_t *ctx = (gcm_ctx_t *)&aes_ctx;
+
+ bzero(ctx->gcm_Htable, ctx->gcm_htab_len);
+ kmem_free(ctx->gcm_Htable, ctx->gcm_htab_len);
+ }
+#endif
return (ret);
}
@@ -1209,6 +1219,14 @@ out:
vmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
}
+#ifdef CAN_USE_GCM_ASM
+ if (((gcm_ctx_t *)&aes_ctx)->gcm_Htable != NULL) {
+ gcm_ctx_t *ctx = (gcm_ctx_t *)&aes_ctx;
+
+ bzero(ctx->gcm_Htable, ctx->gcm_htab_len);
+ kmem_free(ctx->gcm_Htable, ctx->gcm_htab_len);
+ }
+#endif
}
return (ret);