diff options
Diffstat (limited to 'sys/crypto/openssl/aarch64/chacha-armv8.S')
| -rw-r--r-- | sys/crypto/openssl/aarch64/chacha-armv8.S | 59 |
1 files changed, 50 insertions, 9 deletions
diff --git a/sys/crypto/openssl/aarch64/chacha-armv8.S b/sys/crypto/openssl/aarch64/chacha-armv8.S index 4f9d6bd372f7..ee32415ad4c3 100644 --- a/sys/crypto/openssl/aarch64/chacha-armv8.S +++ b/sys/crypto/openssl/aarch64/chacha-armv8.S @@ -3,9 +3,11 @@ #ifndef __KERNEL__ .hidden OPENSSL_armcap_P + + #endif -.text +.section .rodata .align 5 .Lsigma: @@ -17,18 +19,19 @@ .byte 67,104,97,67,104,97,50,48,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 .align 2 -.globl ChaCha20_ctr32 -.type ChaCha20_ctr32,%function +.text + +.globl ChaCha20_ctr32_dflt +.type ChaCha20_ctr32_dflt,%function .align 5 -ChaCha20_ctr32: +ChaCha20_ctr32_dflt: AARCH64_SIGN_LINK_REGISTER - cbz x2,.Labort cmp x2,#192 b.lo .Lshort - #ifndef __KERNEL__ adrp x17,OPENSSL_armcap_P ldr w17,[x17,#:lo12:OPENSSL_armcap_P] +.Lcheck_neon: tst w17,#ARMV7_NEON b.ne .LChaCha20_neon #endif @@ -37,7 +40,8 @@ ChaCha20_ctr32: stp x29,x30,[sp,#-96]! add x29,sp,#0 - adr x5,.Lsigma + adrp x5,.Lsigma + add x5,x5,#:lo12:.Lsigma stp x19,x20,[sp,#16] stp x21,x22,[sp,#32] stp x23,x24,[sp,#48] @@ -304,6 +308,41 @@ ChaCha20_ctr32: ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret +.size ChaCha20_ctr32_dflt,.-ChaCha20_ctr32_dflt + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: + AARCH64_SIGN_LINK_REGISTER + cbz x2,.Labort + cmp x2,#192 + b.lo .Lshort +#ifndef __KERNEL__ + adrp x17,OPENSSL_armcap_P + ldr w17,[x17,#:lo12:OPENSSL_armcap_P] + tst w17,#ARMV8_SVE + b.eq .Lcheck_neon + stp x29,x30,[sp,#-16]! + sub sp,sp,#16 + // SVE handling will inevitably increment the counter + // Neon/Scalar code that follows to process tail data needs to + // use new counter, unfortunately the input counter buffer + // pointed to by ctr is meant to be read-only per API contract + // we have to copy the buffer to stack to be writable by SVE + ldp x5,x6,[x4] + stp x5,x6,[sp] + mov x4,sp + bl ChaCha20_ctr32_sve + cbz x2,1f + bl ChaCha20_ctr32_dflt +1: + add sp,sp,#16 + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +#endif + b .Lshort .size ChaCha20_ctr32,.-ChaCha20_ctr32 #ifdef __KERNEL__ @@ -317,7 +356,8 @@ ChaCha20_neon: stp x29,x30,[sp,#-96]! add x29,sp,#0 - adr x5,.Lsigma + adrp x5,.Lsigma + add x5,x5,#:lo12:.Lsigma stp x19,x20,[sp,#16] stp x21,x22,[sp,#32] stp x23,x24,[sp,#48] @@ -890,7 +930,8 @@ ChaCha20_512_neon: stp x29,x30,[sp,#-96]! add x29,sp,#0 - adr x5,.Lsigma + adrp x5,.Lsigma + add x5,x5,#:lo12:.Lsigma stp x19,x20,[sp,#16] stp x21,x22,[sp,#32] stp x23,x24,[sp,#48] |