libbb/sha1: use SSE2 in unrolled x86-64 code. ~10% faster
function old new delta .rodata 108241 108305 +64 sha1_process_block64 3502 3495 -7 ------------------------------------------------------------------------------ (add/remove: 5/0 grow/shrink: 1/1 up/down: 64/-7) Total: 57 bytes Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>
This commit is contained in:
Denys Vlasenko
parent
1e825acf8d
commit
39369ff460
File diff suppressed because it is too large
Load Diff
@ -6,33 +6,103 @@
|
|||||||
# also contains the diff of the generated file.
|
# also contains the diff of the generated file.
|
||||||
exec >hash_md5_sha_x86-64.S
|
exec >hash_md5_sha_x86-64.S
|
||||||
|
|
||||||
# There is a way to use XMM registers (which always exist for x86-64!) for W[]
|
# Based on http://arctic.org/~dean/crypto/sha1.html.
|
||||||
# For example, if we load W as follows:
|
# ("This SHA1 implementation is public domain.")
|
||||||
# %xmm0: w[0x0] w[0x1] w[0x2] w[0x3]
|
#
|
||||||
# %xmm4: w[0x4] w[0x5] w[0x6] w[0x7]
|
# x86-64 has at least SSE2 vector insns always available.
|
||||||
# %xmm8: w[0x8] w[0x9] w[0xa] w[0xb]
|
# We can use them without any CPUID checks (and without a need
|
||||||
# %xmm12: w[0xc] w[0xd] w[0xe] w[0xf]
|
# for a fallback code if needed insns are not available).
|
||||||
# then the xor'ing operation to generate next W[0..3] is:
|
# This code uses them to calculate W[] ahead of time.
|
||||||
# movaps %xmm0, %xmmT2
|
#
|
||||||
# palignr $0x8, %xmm4, %xmmT2 # form (w[0x2],w[0x3],w[0x4],w[0x5])
|
# Unfortunately, results are passed from vector unit to
|
||||||
# # Right-shifts xmm4:xmmT2 by 8 bytes. Writes shifted result to xmmT2. SSSE3 insn.
|
# integer ALUs on the stack. MOVD/Q insns to move them directly
|
||||||
# movaps %xmm0, %xmmT13
|
# from vector to integer registers are slower than store-to-load
|
||||||
# palignr $0x4,%xmm0,%xmmT13 # form (w[0xd],w[0xe],w[0xf],w[0x0])
|
# forwarding in LSU (on Skylake at least).
|
||||||
# xmm0 = xmm0 ^ t2 ^ xmm8 ^ t13
|
#
|
||||||
# xmm0 = rol32(xmm0,1) # no such insn, have to use pslld+psrld+or
|
# The win against a purely integer code is small on Skylake,
|
||||||
# and then results can be extracted for use:
|
# only about 7-8%. We offload about 1/3 of our operations to the vector unit.
|
||||||
# movd %xmm0, %esi # new W[0]
|
# It can do 4 ops at once in one 128-bit register,
|
||||||
# pextrd $1, %xmm0, %esi # new W[1]
|
# but we have to use x2 of them because of W[0] complication,
|
||||||
# # SSE4.1 insn. Can use EXTRACTPS (also SSE4.1)
|
# SSE2 has no "rotate each word by N bits" insns,
|
||||||
# pextrd $2, %xmm0, %esi # new W[2]
|
# moving data to/from vector unit is clunky, and Skylake
|
||||||
# pextrd $3, %xmm0, %esi # new W[3]
|
# has four integer ALUs unified with three vector ALUs,
|
||||||
# ... but this requires SSE4.1 and SSSE3, which are not universally available on x86-64.
|
# which makes pure integer code rather fast, and makes
|
||||||
|
# vector ops compete with integer ones.
|
||||||
|
#
|
||||||
|
# Zen3, with its separate vector ALUs, wins more, about 12%.
|
||||||
|
|
||||||
|
xmmT1="%xmm4"
|
||||||
|
xmmT2="%xmm5"
|
||||||
|
xmmRCONST="%xmm6"
|
||||||
|
T=`printf '\t'`
|
||||||
|
|
||||||
|
# SSE instructions are longer than 4 bytes on average.
|
||||||
|
# Intel CPUs (up to Tiger Lake at least) can't decode
|
||||||
|
# more than 16 bytes of code in one cycle.
|
||||||
|
# By interleaving SSE code and integer code
|
||||||
|
# we mostly achieve a situation where 16-byte decode fetch window
|
||||||
|
# contains 4 (or more) insns.
|
||||||
|
#
|
||||||
|
# However. On Skylake, there was no observed difference,
|
||||||
|
# but on Zen3, non-interleaved code is ~3% faster
|
||||||
|
# (822 Mb/s versus 795 Mb/s hashing speed).
|
||||||
|
# Off for now:
|
||||||
|
interleave=false
|
||||||
|
|
||||||
|
INTERLEAVE() {
|
||||||
|
$interleave || \
|
||||||
|
{
|
||||||
|
# Generate non-interleaved code
|
||||||
|
# (it should work correctly too)
|
||||||
|
echo "$1"
|
||||||
|
echo "$2"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
(
|
||||||
|
echo "$1" | grep -v '^$' >"$0.temp1"
|
||||||
|
echo "$2" | grep -v '^$' >"$0.temp2"
|
||||||
|
exec 3<"$0.temp1"
|
||||||
|
exec 4<"$0.temp2"
|
||||||
|
IFS=''
|
||||||
|
while :; do
|
||||||
|
line1=''
|
||||||
|
line2=''
|
||||||
|
while :; do
|
||||||
|
read -r line1 <&3
|
||||||
|
if test "${line1:0:1}" != "#" && test "${line1:0:2}" != "$T#"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "$line1"
|
||||||
|
done
|
||||||
|
while :; do
|
||||||
|
read -r line2 <&4
|
||||||
|
if test "${line2:0:4}" = "${T}lea"; then
|
||||||
|
# We use 7-8 byte long forms of LEA.
|
||||||
|
# Do not interleave them with SSE insns
|
||||||
|
# which are also long.
|
||||||
|
echo "$line2"
|
||||||
|
read -r line2 <&4
|
||||||
|
echo "$line2"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if test "${line2:0:1}" != "#" && test "${line2:0:2}" != "$T#"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "$line2"
|
||||||
|
done
|
||||||
|
test "$line1$line2" || break
|
||||||
|
echo "$line1"
|
||||||
|
echo "$line2"
|
||||||
|
done
|
||||||
|
rm "$0.temp1" "$0.temp2"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
echo \
|
echo \
|
||||||
'### Generated by hash_md5_sha_x86-64.S.sh ###
|
"### Generated by hash_md5_sha_x86-64.S.sh ###
|
||||||
|
|
||||||
#if CONFIG_SHA1_SMALL == 0 && defined(__GNUC__) && defined(__x86_64__)
|
#if CONFIG_SHA1_SMALL == 0 && defined(__GNUC__) && defined(__x86_64__)
|
||||||
.section .text.sha1_process_block64,"ax",@progbits
|
.section .text.sha1_process_block64,\"ax\",@progbits
|
||||||
.globl sha1_process_block64
|
.globl sha1_process_block64
|
||||||
.hidden sha1_process_block64
|
.hidden sha1_process_block64
|
||||||
.type sha1_process_block64, @function
|
.type sha1_process_block64, @function
|
||||||
@ -51,16 +121,10 @@ sha1_process_block64:
|
|||||||
# eax..edx: a..d
|
# eax..edx: a..d
|
||||||
# ebp: e
|
# ebp: e
|
||||||
# esi,edi: temps
|
# esi,edi: temps
|
||||||
# -32+4*n(%rsp),r8...r15: W[0..7,8..15]
|
# xmm0..xmm3: W[]
|
||||||
# (TODO: actually W[0..7] are used a bit more often, put _them_ into r8..r15?)
|
# xmm4,xmm5: temps
|
||||||
movl $3, %eax
|
# xmm6: current round constant
|
||||||
1:
|
# -64(%rsp): area for passing RCONST + W[] from vector to integer units
|
||||||
movq (%rdi,%rax,8), %rsi
|
|
||||||
bswapq %rsi
|
|
||||||
rolq $32, %rsi
|
|
||||||
movq %rsi, -32(%rsp,%rax,8)
|
|
||||||
decl %eax
|
|
||||||
jns 1b
|
|
||||||
|
|
||||||
movl 80(%rdi), %eax # a = ctx->hash[0]
|
movl 80(%rdi), %eax # a = ctx->hash[0]
|
||||||
movl 84(%rdi), %ebx # b = ctx->hash[1]
|
movl 84(%rdi), %ebx # b = ctx->hash[1]
|
||||||
@ -68,32 +132,120 @@ sha1_process_block64:
|
|||||||
movl 92(%rdi), %edx # d = ctx->hash[3]
|
movl 92(%rdi), %edx # d = ctx->hash[3]
|
||||||
movl 96(%rdi), %ebp # e = ctx->hash[4]
|
movl 96(%rdi), %ebp # e = ctx->hash[4]
|
||||||
|
|
||||||
|
movaps rconst0x5A827999(%rip), $xmmRCONST
|
||||||
|
|
||||||
|
# For round 1, steps 0 and 8..15, we pass W[0,8..15] in esi,r8..r15
|
||||||
|
# instead of spilling them to stack.
|
||||||
|
# (We lose parallelized addition of RCONST, but LEA
|
||||||
|
# can do two additions at once, so...)
|
||||||
|
movq 4*0(%rdi), %rsi
|
||||||
|
movq 4*2(%rdi), %r10
|
||||||
|
bswapq %rsi
|
||||||
|
bswapq %r10
|
||||||
|
rolq \$32, %rsi # rsi = W[1]:W[0]
|
||||||
|
rolq \$32, %r10
|
||||||
|
movq %rsi, %xmm0
|
||||||
|
movq %r10, $xmmT1
|
||||||
|
punpcklqdq $xmmT1, %xmm0 # xmm0 = r10:rsi = (W[0],W[1],W[2],W[3])
|
||||||
|
movaps %xmm0, $xmmT1
|
||||||
|
paddd $xmmRCONST, $xmmT1
|
||||||
|
movups $xmmT1, -64+4*0(%rsp)
|
||||||
|
|
||||||
|
movq 4*4(%rdi), %r8
|
||||||
|
movq 4*6(%rdi), %r10
|
||||||
|
bswapq %r8
|
||||||
|
bswapq %r10
|
||||||
|
rolq \$32, %r8
|
||||||
|
rolq \$32, %r10
|
||||||
|
movq %r8, %xmm1
|
||||||
|
movq %r10, $xmmT1
|
||||||
|
punpcklqdq $xmmT1, %xmm1 # xmm1 = r10:r8 = (W[4],W[5],W[6],W[7])
|
||||||
|
movaps %xmm1, $xmmT1
|
||||||
|
paddd $xmmRCONST, $xmmT1
|
||||||
|
movups $xmmT1, -64+4*4(%rsp)
|
||||||
|
|
||||||
movq 4*8(%rdi), %r8
|
movq 4*8(%rdi), %r8
|
||||||
movq 4*10(%rdi), %r10
|
movq 4*10(%rdi), %r10
|
||||||
bswapq %r8
|
bswapq %r8
|
||||||
bswapq %r10
|
bswapq %r10
|
||||||
|
movl %r8d, %r9d # r9d = W[9]
|
||||||
|
rolq \$32, %r8 # r8 = W[9]:W[8]
|
||||||
|
movl %r10d, %r11d # r11d = W[11]
|
||||||
|
rolq \$32, %r10 # r10 = W[11]:W[10]
|
||||||
|
movq %r8, %xmm2
|
||||||
|
movq %r10, $xmmT1
|
||||||
|
punpcklqdq $xmmT1, %xmm2 # xmm2 = r10:r8 = (W[8],W[9],W[10],W[11])
|
||||||
|
|
||||||
movq 4*12(%rdi), %r12
|
movq 4*12(%rdi), %r12
|
||||||
movq 4*14(%rdi), %r14
|
movq 4*14(%rdi), %r14
|
||||||
bswapq %r12
|
bswapq %r12
|
||||||
bswapq %r14
|
bswapq %r14
|
||||||
movl %r8d, %r9d
|
movl %r12d, %r13d # r13d = W[13]
|
||||||
shrq $32, %r8
|
rolq \$32, %r12 # r12 = W[13]:W[12]
|
||||||
movl %r10d, %r11d
|
movl %r14d, %r15d # r15d = W[15]
|
||||||
shrq $32, %r10
|
rolq \$32, %r14 # r14 = W[15]:W[14]
|
||||||
movl %r12d, %r13d
|
movq %r12, %xmm3
|
||||||
shrq $32, %r12
|
movq %r14, $xmmT1
|
||||||
movl %r14d, %r15d
|
punpcklqdq $xmmT1, %xmm3 # xmm3 = r14:r12 = (W[12],W[13],W[14],W[15])
|
||||||
shrq $32, %r14
|
"
|
||||||
'
|
|
||||||
W32() {
|
PREP() {
|
||||||
test "$1" || exit 1
|
local xmmW0=$1
|
||||||
test "$1" -lt 0 && exit 1
|
local xmmW4=$2
|
||||||
test "$1" -gt 15 && exit 1
|
local xmmW8=$3
|
||||||
test "$1" -lt 8 && echo "-32+4*$1(%rsp)"
|
local xmmW12=$4
|
||||||
test "$1" -ge 8 && echo "%r${1}d"
|
# the above must be %xmm0..3 in some permutation
|
||||||
|
local dstmem=$5
|
||||||
|
#W[0] = rol(W[13] ^ W[8] ^ W[2] ^ W[0], 1);
|
||||||
|
#W[1] = rol(W[14] ^ W[9] ^ W[3] ^ W[1], 1);
|
||||||
|
#W[2] = rol(W[15] ^ W[10] ^ W[4] ^ W[2], 1);
|
||||||
|
#W[3] = rol( 0 ^ W[11] ^ W[5] ^ W[3], 1);
|
||||||
|
#W[3] ^= rol(W[0], 1);
|
||||||
|
echo "# PREP $@
|
||||||
|
movaps $xmmW12, $xmmT1
|
||||||
|
psrldq \$4, $xmmT1 # rshift by 4 bytes: T1 = ([13],[14],[15],0)
|
||||||
|
|
||||||
|
pshufd \$0x4e, $xmmW0, $xmmT2 # 01001110=2,3,0,1 shuffle, ([2],[3],x,x)
|
||||||
|
punpcklqdq $xmmW4, $xmmT2 # T2 = W4[0..63]:T2[0..63] = ([2],[3],[4],[5])
|
||||||
|
|
||||||
|
xorps $xmmW8, $xmmW0 # ([8],[9],[10],[11]) ^ ([0],[1],[2],[3])
|
||||||
|
xorps $xmmT1, $xmmT2 # ([13],[14],[15],0) ^ ([2],[3],[4],[5])
|
||||||
|
xorps $xmmT2, $xmmW0 # ^
|
||||||
|
# W0 = unrotated (W[0]..W[3]), still needs W[3] fixup
|
||||||
|
movaps $xmmW0, $xmmT2
|
||||||
|
|
||||||
|
xorps $xmmT1, $xmmT1 # rol(W0,1):
|
||||||
|
pcmpgtd $xmmW0, $xmmT1 # ffffffff for elements <0 (ones with msb bit 1)
|
||||||
|
paddd $xmmW0, $xmmW0 # shift left by 1
|
||||||
|
psubd $xmmT1, $xmmW0 # add 1 to those who had msb bit 1
|
||||||
|
# W0 = rotated (W[0]..W[3]), still needs W[3] fixup
|
||||||
|
|
||||||
|
pslldq \$12, $xmmT2 # lshift by 12 bytes: T2 = (0,0,0,unrotW[0])
|
||||||
|
movaps $xmmT2, $xmmT1
|
||||||
|
pslld \$2, $xmmT2
|
||||||
|
psrld \$30, $xmmT1
|
||||||
|
# xorps $xmmT1, $xmmT2 # rol((0,0,0,unrotW[0]),2)
|
||||||
|
xorps $xmmT1, $xmmW0 # same result, but does not depend on/does not modify T2
|
||||||
|
|
||||||
|
xorps $xmmT2, $xmmW0 # W0 = rol(W[0]..W[3],1) ^ (0,0,0,rol(unrotW[0],2))
|
||||||
|
"
|
||||||
|
# movq $xmmW0, %r8 # high latency (~6 cycles)
|
||||||
|
# movaps $xmmW0, $xmmT1
|
||||||
|
# psrldq \$8, $xmmT1 # rshift by 8 bytes: move upper 64 bits to lower
|
||||||
|
# movq $xmmT1, %r10 # high latency
|
||||||
|
# movq %r8, %r9
|
||||||
|
# movq %r10, %r11
|
||||||
|
# shrq \$32, %r9
|
||||||
|
# shrq \$32, %r11
|
||||||
|
# ^^^ slower than passing the results on stack (!!!)
|
||||||
|
echo "
|
||||||
|
movaps $xmmW0, $xmmT2
|
||||||
|
paddd $xmmRCONST, $xmmT2
|
||||||
|
movups $xmmT2, $dstmem
|
||||||
|
"
|
||||||
}
|
}
|
||||||
|
|
||||||
# It's possible to interleave insns in rounds to mostly eliminate
|
# It's possible to interleave integer insns in rounds to mostly eliminate
|
||||||
# dependency chains, but this likely to only help old Pentium-based
|
# dependency chains, but this likely to only help old Pentium-based
|
||||||
# CPUs (ones without OOO, which can only simultaneously execute a pair
|
# CPUs (ones without OOO, which can only simultaneously execute a pair
|
||||||
# of _adjacent_ insns).
|
# of _adjacent_ insns).
|
||||||
@ -107,21 +259,16 @@ local n0=$(((n+0) & 15))
|
|||||||
echo "
|
echo "
|
||||||
# $n
|
# $n
|
||||||
";test $n0 = 0 && echo "
|
";test $n0 = 0 && echo "
|
||||||
# W[0], already in %esi
|
leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n]
|
||||||
";test $n0 != 0 && test $n0 -lt 8 && echo "
|
";test $n0 != 0 && test $n0 -lt 8 && echo "
|
||||||
movl `W32 $n0`, %esi # W[n]
|
addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n]
|
||||||
";test $n0 -ge 8 && echo "
|
";test $n0 -ge 8 && echo "
|
||||||
# W[n], in %r$n0
|
leal $RCONST(%r$e,%r$n0), %e$e # e += RCONST + W[n]
|
||||||
";echo "
|
";echo "
|
||||||
movl %e$c, %edi # c
|
movl %e$c, %edi # c
|
||||||
xorl %e$d, %edi # ^d
|
xorl %e$d, %edi # ^d
|
||||||
andl %e$b, %edi # &b
|
andl %e$b, %edi # &b
|
||||||
xorl %e$d, %edi # (((c ^ d) & b) ^ d)
|
xorl %e$d, %edi # (((c ^ d) & b) ^ d)
|
||||||
";test $n0 -lt 8 && echo "
|
|
||||||
leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
leal $RCONST(%r$e,%r$n0), %e$e # e += RCONST + W[n]
|
|
||||||
";echo "
|
|
||||||
addl %edi, %e$e # e += (((c ^ d) & b) ^ d)
|
addl %edi, %e$e # e += (((c ^ d) & b) ^ d)
|
||||||
movl %e$a, %esi #
|
movl %e$a, %esi #
|
||||||
roll \$5, %esi # rotl32(a,5)
|
roll \$5, %esi # rotl32(a,5)
|
||||||
@ -138,28 +285,11 @@ local n2=$(((n+2) & 15))
|
|||||||
local n0=$(((n+0) & 15))
|
local n0=$(((n+0) & 15))
|
||||||
echo "
|
echo "
|
||||||
# $n
|
# $n
|
||||||
";test $n0 -lt 8 && echo "
|
|
||||||
movl `W32 $n13`, %esi # W[(n+13) & 15]
|
|
||||||
xorl `W32 $n8`, %esi # ^W[(n+8) & 15]
|
|
||||||
xorl `W32 $n2`, %esi # ^W[(n+2) & 15]
|
|
||||||
xorl `W32 $n0`, %esi # ^W[n & 15]
|
|
||||||
roll %esi #
|
|
||||||
movl %esi, `W32 $n0` # store to W[n & 15]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
xorl `W32 $n13`, `W32 $n0` # W[n & 15] ^= W[(n+13) & 15]
|
|
||||||
xorl `W32 $n8`, `W32 $n0` # ^W[(n+8) & 15]
|
|
||||||
xorl `W32 $n2`, `W32 $n0` # ^W[(n+2) & 15]
|
|
||||||
roll `W32 $n0` #
|
|
||||||
";echo "
|
|
||||||
movl %e$c, %edi # c
|
movl %e$c, %edi # c
|
||||||
xorl %e$d, %edi # ^d
|
xorl %e$d, %edi # ^d
|
||||||
andl %e$b, %edi # &b
|
andl %e$b, %edi # &b
|
||||||
xorl %e$d, %edi # (((c ^ d) & b) ^ d)
|
xorl %e$d, %edi # (((c ^ d) & b) ^ d)
|
||||||
";test $n0 -lt 8 && echo "
|
addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n & 15]
|
||||||
leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n & 15]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
leal $RCONST(%r$e,%r$n0), %e$e # e += RCONST + W[n & 15]
|
|
||||||
";echo "
|
|
||||||
addl %edi, %e$e # e += (((c ^ d) & b) ^ d)
|
addl %edi, %e$e # e += (((c ^ d) & b) ^ d)
|
||||||
movl %e$a, %esi #
|
movl %e$a, %esi #
|
||||||
roll \$5, %esi # rotl32(a,5)
|
roll \$5, %esi # rotl32(a,5)
|
||||||
@ -167,13 +297,6 @@ echo "
|
|||||||
rorl \$2, %e$b # b = rotl32(b,30)
|
rorl \$2, %e$b # b = rotl32(b,30)
|
||||||
"
|
"
|
||||||
}
|
}
|
||||||
{
|
|
||||||
RCONST=0x5A827999
|
|
||||||
RD1A ax bx cx dx bp 0; RD1A bp ax bx cx dx 1; RD1A dx bp ax bx cx 2; RD1A cx dx bp ax bx 3; RD1A bx cx dx bp ax 4
|
|
||||||
RD1A ax bx cx dx bp 5; RD1A bp ax bx cx dx 6; RD1A dx bp ax bx cx 7; RD1A cx dx bp ax bx 8; RD1A bx cx dx bp ax 9
|
|
||||||
RD1A ax bx cx dx bp 10; RD1A bp ax bx cx dx 11; RD1A dx bp ax bx cx 12; RD1A cx dx bp ax bx 13; RD1A bx cx dx bp ax 14
|
|
||||||
RD1A ax bx cx dx bp 15; RD1B bp ax bx cx dx 16; RD1B dx bp ax bx cx 17; RD1B cx dx bp ax bx 18; RD1B bx cx dx bp ax 19
|
|
||||||
} | grep -v '^$'
|
|
||||||
|
|
||||||
RD2() {
|
RD2() {
|
||||||
local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
|
local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
|
||||||
@ -184,27 +307,10 @@ local n2=$(((n+2) & 15))
|
|||||||
local n0=$(((n+0) & 15))
|
local n0=$(((n+0) & 15))
|
||||||
echo "
|
echo "
|
||||||
# $n
|
# $n
|
||||||
";test $n0 -lt 8 && echo "
|
|
||||||
movl `W32 $n13`, %esi # W[(n+13) & 15]
|
|
||||||
xorl `W32 $n8`, %esi # ^W[(n+8) & 15]
|
|
||||||
xorl `W32 $n2`, %esi # ^W[(n+2) & 15]
|
|
||||||
xorl `W32 $n0`, %esi # ^W[n & 15]
|
|
||||||
roll %esi #
|
|
||||||
movl %esi, `W32 $n0` # store to W[n & 15]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
xorl `W32 $n13`, `W32 $n0` # W[n & 15] ^= W[(n+13) & 15]
|
|
||||||
xorl `W32 $n8`, `W32 $n0` # ^W[(n+8) & 15]
|
|
||||||
xorl `W32 $n2`, `W32 $n0` # ^W[(n+2) & 15]
|
|
||||||
roll `W32 $n0` #
|
|
||||||
";echo "
|
|
||||||
movl %e$c, %edi # c
|
movl %e$c, %edi # c
|
||||||
xorl %e$d, %edi # ^d
|
xorl %e$d, %edi # ^d
|
||||||
xorl %e$b, %edi # ^b
|
xorl %e$b, %edi # ^b
|
||||||
";test $n0 -lt 8 && echo "
|
addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n & 15]
|
||||||
leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n & 15]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
leal $RCONST(%r$e,%r$n0), %e$e # e += RCONST + W[n & 15]
|
|
||||||
";echo "
|
|
||||||
addl %edi, %e$e # e += (c ^ d ^ b)
|
addl %edi, %e$e # e += (c ^ d ^ b)
|
||||||
movl %e$a, %esi #
|
movl %e$a, %esi #
|
||||||
roll \$5, %esi # rotl32(a,5)
|
roll \$5, %esi # rotl32(a,5)
|
||||||
@ -212,13 +318,6 @@ echo "
|
|||||||
rorl \$2, %e$b # b = rotl32(b,30)
|
rorl \$2, %e$b # b = rotl32(b,30)
|
||||||
"
|
"
|
||||||
}
|
}
|
||||||
{
|
|
||||||
RCONST=0x6ED9EBA1
|
|
||||||
RD2 ax bx cx dx bp 20; RD2 bp ax bx cx dx 21; RD2 dx bp ax bx cx 22; RD2 cx dx bp ax bx 23; RD2 bx cx dx bp ax 24
|
|
||||||
RD2 ax bx cx dx bp 25; RD2 bp ax bx cx dx 26; RD2 dx bp ax bx cx 27; RD2 cx dx bp ax bx 28; RD2 bx cx dx bp ax 29
|
|
||||||
RD2 ax bx cx dx bp 30; RD2 bp ax bx cx dx 31; RD2 dx bp ax bx cx 32; RD2 cx dx bp ax bx 33; RD2 bx cx dx bp ax 34
|
|
||||||
RD2 ax bx cx dx bp 35; RD2 bp ax bx cx dx 36; RD2 dx bp ax bx cx 37; RD2 cx dx bp ax bx 38; RD2 bx cx dx bp ax 39
|
|
||||||
} | grep -v '^$'
|
|
||||||
|
|
||||||
RD3() {
|
RD3() {
|
||||||
local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
|
local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
|
||||||
@ -235,53 +334,82 @@ echo "
|
|||||||
andl %e$c, %esi # si: b & c
|
andl %e$c, %esi # si: b & c
|
||||||
andl %e$d, %edi # di: (b | c) & d
|
andl %e$d, %edi # di: (b | c) & d
|
||||||
orl %esi, %edi # ((b | c) & d) | (b & c)
|
orl %esi, %edi # ((b | c) & d) | (b & c)
|
||||||
";test $n0 -lt 8 && echo "
|
|
||||||
movl `W32 $n13`, %esi # W[(n+13) & 15]
|
|
||||||
xorl `W32 $n8`, %esi # ^W[(n+8) & 15]
|
|
||||||
xorl `W32 $n2`, %esi # ^W[(n+2) & 15]
|
|
||||||
xorl `W32 $n0`, %esi # ^W[n & 15]
|
|
||||||
roll %esi #
|
|
||||||
movl %esi, `W32 $n0` # store to W[n & 15]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
xorl `W32 $n13`, `W32 $n0` # W[n & 15] ^= W[(n+13) & 15]
|
|
||||||
xorl `W32 $n8`, `W32 $n0` # ^W[(n+8) & 15]
|
|
||||||
xorl `W32 $n2`, `W32 $n0` # ^W[(n+2) & 15]
|
|
||||||
roll `W32 $n0` #
|
|
||||||
";echo "
|
|
||||||
addl %edi, %e$e # += ((b | c) & d) | (b & c)
|
addl %edi, %e$e # += ((b | c) & d) | (b & c)
|
||||||
";test $n0 -lt 8 && echo "
|
addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n & 15]
|
||||||
leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n & 15]
|
|
||||||
";test $n0 -ge 8 && echo "
|
|
||||||
leal $RCONST(%r$e,%r$n0), %e$e # e += RCONST + W[n & 15]
|
|
||||||
";echo "
|
|
||||||
movl %e$a, %esi #
|
movl %e$a, %esi #
|
||||||
roll \$5, %esi # rotl32(a,5)
|
roll \$5, %esi # rotl32(a,5)
|
||||||
addl %esi, %e$e # e += rotl32(a,5)
|
addl %esi, %e$e # e += rotl32(a,5)
|
||||||
rorl \$2, %e$b # b = rotl32(b,30)
|
rorl \$2, %e$b # b = rotl32(b,30)
|
||||||
"
|
"
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
#RCONST=0x8F1BBCDC "out of range for signed 32bit displacement"
|
# Round 1
|
||||||
RCONST=-0x70E44324
|
RCONST=0x5A827999
|
||||||
RD3 ax bx cx dx bp 40; RD3 bp ax bx cx dx 41; RD3 dx bp ax bx cx 42; RD3 cx dx bp ax bx 43; RD3 bx cx dx bp ax 44
|
RD1A ax bx cx dx bp 0; RD1A bp ax bx cx dx 1; RD1A dx bp ax bx cx 2; RD1A cx dx bp ax bx 3;
|
||||||
RD3 ax bx cx dx bp 45; RD3 bp ax bx cx dx 46; RD3 dx bp ax bx cx 47; RD3 cx dx bp ax bx 48; RD3 bx cx dx bp ax 49
|
RD1A bx cx dx bp ax 4; RD1A ax bx cx dx bp 5; RD1A bp ax bx cx dx 6; RD1A dx bp ax bx cx 7;
|
||||||
RD3 ax bx cx dx bp 50; RD3 bp ax bx cx dx 51; RD3 dx bp ax bx cx 52; RD3 cx dx bp ax bx 53; RD3 bx cx dx bp ax 54
|
a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
|
||||||
RD3 ax bx cx dx bp 55; RD3 bp ax bx cx dx 56; RD3 dx bp ax bx cx 57; RD3 cx dx bp ax bx 58; RD3 bx cx dx bp ax 59
|
b=`RD1A cx dx bp ax bx 8; RD1A bx cx dx bp ax 9; RD1A ax bx cx dx bp 10; RD1A bp ax bx cx dx 11;`
|
||||||
} | grep -v '^$'
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`echo " movaps rconst0x6ED9EBA1(%rip), $xmmRCONST"
|
||||||
|
PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
|
||||||
|
b=`RD1A dx bp ax bx cx 12; RD1A cx dx bp ax bx 13; RD1A bx cx dx bp ax 14; RD1A ax bx cx dx bp 15;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
|
||||||
|
b=`RD1B bp ax bx cx dx 16; RD1B dx bp ax bx cx 17; RD1B cx dx bp ax bx 18; RD1B bx cx dx bp ax 19;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
|
||||||
|
# Round 2
|
||||||
|
RCONST=0x6ED9EBA1
|
||||||
|
a=`PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
|
||||||
|
b=`RD2 ax bx cx dx bp 20; RD2 bp ax bx cx dx 21; RD2 dx bp ax bx cx 22; RD2 cx dx bp ax bx 23;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
|
||||||
|
b=`RD2 bx cx dx bp ax 24; RD2 ax bx cx dx bp 25; RD2 bp ax bx cx dx 26; RD2 dx bp ax bx cx 27;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
|
||||||
|
b=`RD2 cx dx bp ax bx 28; RD2 bx cx dx bp ax 29; RD2 ax bx cx dx bp 30; RD2 bp ax bx cx dx 31;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`echo " movaps rconst0x8F1BBCDC(%rip), $xmmRCONST"
|
||||||
|
PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
|
||||||
|
b=`RD2 dx bp ax bx cx 32; RD2 cx dx bp ax bx 33; RD2 bx cx dx bp ax 34; RD2 ax bx cx dx bp 35;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
|
||||||
|
b=`RD2 bp ax bx cx dx 36; RD2 dx bp ax bx cx 37; RD2 cx dx bp ax bx 38; RD2 bx cx dx bp ax 39;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
|
||||||
|
# Round 3
|
||||||
|
RCONST=0x8F1BBCDC
|
||||||
|
a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
|
||||||
|
b=`RD3 ax bx cx dx bp 40; RD3 bp ax bx cx dx 41; RD3 dx bp ax bx cx 42; RD3 cx dx bp ax bx 43;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
|
||||||
|
b=`RD3 bx cx dx bp ax 44; RD3 ax bx cx dx bp 45; RD3 bp ax bx cx dx 46; RD3 dx bp ax bx cx 47;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
|
||||||
|
b=`RD3 cx dx bp ax bx 48; RD3 bx cx dx bp ax 49; RD3 ax bx cx dx bp 50; RD3 bp ax bx cx dx 51;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`echo " movaps rconst0xCA62C1D6(%rip), $xmmRCONST"
|
||||||
|
PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
|
||||||
|
b=`RD3 dx bp ax bx cx 52; RD3 cx dx bp ax bx 53; RD3 bx cx dx bp ax 54; RD3 ax bx cx dx bp 55;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
|
||||||
|
b=`RD3 bp ax bx cx dx 56; RD3 dx bp ax bx cx 57; RD3 cx dx bp ax bx 58; RD3 bx cx dx bp ax 59;`
|
||||||
|
INTERLEAVE "$a" "$b"
|
||||||
|
|
||||||
# Round 4 has the same logic as round 2, only n and RCONST are different
|
# Round 4 has the same logic as round 2, only n and RCONST are different
|
||||||
{
|
RCONST=0xCA62C1D6
|
||||||
#RCONST=0xCA62C1D6 "out of range for signed 32bit displacement"
|
a=`PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
|
||||||
RCONST=-0x359D3E2A
|
b=`RD2 ax bx cx dx bp 60; RD2 bp ax bx cx dx 61; RD2 dx bp ax bx cx 62; RD2 cx dx bp ax bx 63;`
|
||||||
RD2 ax bx cx dx bp 60; RD2 bp ax bx cx dx 61; RD2 dx bp ax bx cx 62; RD2 cx dx bp ax bx 63; RD2 bx cx dx bp ax 64
|
INTERLEAVE "$a" "$b"
|
||||||
RD2 ax bx cx dx bp 65; RD2 bp ax bx cx dx 66; RD2 dx bp ax bx cx 67; RD2 cx dx bp ax bx 68; RD2 bx cx dx bp ax 69
|
a=`PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
|
||||||
RD2 ax bx cx dx bp 70; RD2 bp ax bx cx dx 71; RD2 dx bp ax bx cx 72; RD2 cx dx bp ax bx 73; RD2 bx cx dx bp ax 74
|
b=`RD2 bx cx dx bp ax 64; RD2 ax bx cx dx bp 65; RD2 bp ax bx cx dx 66; RD2 dx bp ax bx cx 67;`
|
||||||
RD2 ax bx cx dx bp 75; RD2 bp ax bx cx dx 76; RD2 dx bp ax bx cx 77; RD2 cx dx bp ax bx 78; RD2 bx cx dx bp ax 79
|
INTERLEAVE "$a" "$b"
|
||||||
# Note: new W[n&15] values generated in last 3 iterations
|
a=`PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
|
||||||
# (W[13,14,15]) are unused after each of these iterations.
|
b=`RD2 cx dx bp ax bx 68; RD2 bx cx dx bp ax 69; RD2 ax bx cx dx bp 70; RD2 bp ax bx cx dx 71;`
|
||||||
# Since we use r8..r15 for W[8..15], this does not matter.
|
INTERLEAVE "$a" "$b"
|
||||||
# If we switch to e.g. using r8..r15 for W[0..7], then saving of W[13,14,15]
|
RD2 dx bp ax bx cx 72; RD2 cx dx bp ax bx 73; RD2 bx cx dx bp ax 74; RD2 ax bx cx dx bp 75;
|
||||||
# (the "movl %esi, `W32 $n0`" insn) is a dead store and can be removed.
|
RD2 bp ax bx cx dx 76; RD2 dx bp ax bx cx 77; RD2 cx dx bp ax bx 78; RD2 bx cx dx bp ax 79;
|
||||||
} | grep -v '^$'
|
} | grep -v '^$'
|
||||||
|
|
||||||
echo "
|
echo "
|
||||||
@ -300,4 +428,28 @@ echo "
|
|||||||
|
|
||||||
ret
|
ret
|
||||||
.size sha1_process_block64, .-sha1_process_block64
|
.size sha1_process_block64, .-sha1_process_block64
|
||||||
|
|
||||||
|
.section .rodata.cst16.sha1const, \"aM\", @progbits, 16
|
||||||
|
.align 16
|
||||||
|
rconst0x5A827999:
|
||||||
|
.long 0x5A827999
|
||||||
|
.long 0x5A827999
|
||||||
|
.long 0x5A827999
|
||||||
|
.long 0x5A827999
|
||||||
|
rconst0x6ED9EBA1:
|
||||||
|
.long 0x6ED9EBA1
|
||||||
|
.long 0x6ED9EBA1
|
||||||
|
.long 0x6ED9EBA1
|
||||||
|
.long 0x6ED9EBA1
|
||||||
|
rconst0x8F1BBCDC:
|
||||||
|
.long 0x8F1BBCDC
|
||||||
|
.long 0x8F1BBCDC
|
||||||
|
.long 0x8F1BBCDC
|
||||||
|
.long 0x8F1BBCDC
|
||||||
|
rconst0xCA62C1D6:
|
||||||
|
.long 0xCA62C1D6
|
||||||
|
.long 0xCA62C1D6
|
||||||
|
.long 0xCA62C1D6
|
||||||
|
.long 0xCA62C1D6
|
||||||
|
|
||||||
#endif"
|
#endif"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user