diff options
Diffstat (limited to 'contrib/bearssl/src/symcipher/chacha20_sse2.c')
| -rw-r--r-- | contrib/bearssl/src/symcipher/chacha20_sse2.c | 237 |
1 files changed, 237 insertions, 0 deletions
diff --git a/contrib/bearssl/src/symcipher/chacha20_sse2.c b/contrib/bearssl/src/symcipher/chacha20_sse2.c new file mode 100644 index 000000000000..92b4a4a83591 --- /dev/null +++ b/contrib/bearssl/src/symcipher/chacha20_sse2.c @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org> + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define BR_ENABLE_INTRINSICS 1 +#include "inner.h" + +#if BR_SSE2 + +/* + * This file contains a ChaCha20 implementation that leverages SSE2 + * opcodes for better performance. + */ + +/* see bearssl_block.h */ +br_chacha20_run +br_chacha20_sse2_get(void) +{ + /* + * If using 64-bit mode, then SSE2 opcodes should be automatically + * available, since they are part of the ABI. + * + * In 32-bit mode, we use CPUID to detect the SSE2 feature. + */ + +#if BR_amd64 + return &br_chacha20_sse2_run; +#else + + /* + * SSE2 support is indicated by bit 26 in EDX. + */ + if (br_cpuid(0, 0, 0, 0x04000000)) { + return &br_chacha20_sse2_run; + } else { + return 0; + } +#endif +} + +BR_TARGETS_X86_UP + +/* see bearssl_block.h */ +BR_TARGET("sse2") +uint32_t +br_chacha20_sse2_run(const void *key, + const void *iv, uint32_t cc, void *data, size_t len) +{ + unsigned char *buf; + uint32_t ivtmp[4]; + __m128i kw0, kw1; + __m128i iw, cw; + __m128i one; + + static const uint32_t CW[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + + buf = data; + kw0 = _mm_loadu_si128(key); + kw1 = _mm_loadu_si128((const void *)((const unsigned char *)key + 16)); + ivtmp[0] = cc; + memcpy(ivtmp + 1, iv, 12); + iw = _mm_loadu_si128((const void *)ivtmp); + cw = _mm_loadu_si128((const void *)CW); + one = _mm_set_epi32(0, 0, 0, 1); + + while (len > 0) { + /* + * sj contains state words 4*j to 4*j+3. + */ + __m128i s0, s1, s2, s3; + int i; + + s0 = cw; + s1 = kw0; + s2 = kw1; + s3 = iw; + for (i = 0; i < 10; i ++) { + /* + * Even round is straightforward application on + * the state words. + */ + s0 = _mm_add_epi32(s0, s1); + s3 = _mm_xor_si128(s3, s0); + s3 = _mm_or_si128( + _mm_slli_epi32(s3, 16), + _mm_srli_epi32(s3, 16)); + + s2 = _mm_add_epi32(s2, s3); + s1 = _mm_xor_si128(s1, s2); + s1 = _mm_or_si128( + _mm_slli_epi32(s1, 12), + _mm_srli_epi32(s1, 20)); + + s0 = _mm_add_epi32(s0, s1); + s3 = _mm_xor_si128(s3, s0); + s3 = _mm_or_si128( + _mm_slli_epi32(s3, 8), + _mm_srli_epi32(s3, 24)); + + s2 = _mm_add_epi32(s2, s3); + s1 = _mm_xor_si128(s1, s2); + s1 = _mm_or_si128( + _mm_slli_epi32(s1, 7), + _mm_srli_epi32(s1, 25)); + + /* + * For the odd round, we must rotate some state + * words so that the computations apply on the + * right combinations of words. + */ + s1 = _mm_shuffle_epi32(s1, 0x39); + s2 = _mm_shuffle_epi32(s2, 0x4E); + s3 = _mm_shuffle_epi32(s3, 0x93); + + s0 = _mm_add_epi32(s0, s1); + s3 = _mm_xor_si128(s3, s0); + s3 = _mm_or_si128( + _mm_slli_epi32(s3, 16), + _mm_srli_epi32(s3, 16)); + + s2 = _mm_add_epi32(s2, s3); + s1 = _mm_xor_si128(s1, s2); + s1 = _mm_or_si128( + _mm_slli_epi32(s1, 12), + _mm_srli_epi32(s1, 20)); + + s0 = _mm_add_epi32(s0, s1); + s3 = _mm_xor_si128(s3, s0); + s3 = _mm_or_si128( + _mm_slli_epi32(s3, 8), + _mm_srli_epi32(s3, 24)); + + s2 = _mm_add_epi32(s2, s3); + s1 = _mm_xor_si128(s1, s2); + s1 = _mm_or_si128( + _mm_slli_epi32(s1, 7), + _mm_srli_epi32(s1, 25)); + + /* + * After the odd round, we rotate back the values + * to undo the rotate at the start of the odd round. + */ + s1 = _mm_shuffle_epi32(s1, 0x93); + s2 = _mm_shuffle_epi32(s2, 0x4E); + s3 = _mm_shuffle_epi32(s3, 0x39); + } + + /* + * Addition with the initial state. + */ + s0 = _mm_add_epi32(s0, cw); + s1 = _mm_add_epi32(s1, kw0); + s2 = _mm_add_epi32(s2, kw1); + s3 = _mm_add_epi32(s3, iw); + + /* + * Increment block counter. + */ + iw = _mm_add_epi32(iw, one); + + /* + * XOR final state with the data. + */ + if (len < 64) { + unsigned char tmp[64]; + size_t u; + + _mm_storeu_si128((void *)(tmp + 0), s0); + _mm_storeu_si128((void *)(tmp + 16), s1); + _mm_storeu_si128((void *)(tmp + 32), s2); + _mm_storeu_si128((void *)(tmp + 48), s3); + for (u = 0; u < len; u ++) { + buf[u] ^= tmp[u]; + } + break; + } else { + __m128i b0, b1, b2, b3; + + b0 = _mm_loadu_si128((const void *)(buf + 0)); + b1 = _mm_loadu_si128((const void *)(buf + 16)); + b2 = _mm_loadu_si128((const void *)(buf + 32)); + b3 = _mm_loadu_si128((const void *)(buf + 48)); + b0 = _mm_xor_si128(b0, s0); + b1 = _mm_xor_si128(b1, s1); + b2 = _mm_xor_si128(b2, s2); + b3 = _mm_xor_si128(b3, s3); + _mm_storeu_si128((void *)(buf + 0), b0); + _mm_storeu_si128((void *)(buf + 16), b1); + _mm_storeu_si128((void *)(buf + 32), b2); + _mm_storeu_si128((void *)(buf + 48), b3); + buf += 64; + len -= 64; + } + } + + /* + * _mm_extract_epi32() requires SSE4.1. We prefer to stick to + * raw SSE2, thus we use _mm_extract_epi16(). + */ + return (uint32_t)_mm_extract_epi16(iw, 0) + | ((uint32_t)_mm_extract_epi16(iw, 1) << 16); +} + +BR_TARGETS_X86_DOWN + +#else + +/* see bearssl_block.h */ +br_chacha20_run +br_chacha20_sse2_get(void) +{ + return 0; +} + +#endif |