Filippo Valsorda
2dbfe9001f
The new code is meant to be readable without external references for Poly1305, and explains the field logic. The generic code is now 30-50% faster on a Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz, and even better on a 3.1 GHz i7 MacBook. name old time/op new time/op delta 64-48 126ns ± 0% 80ns ± 1% -36.24% (p=0.000 n=16+20) 1K-48 1.07µs ± 0% 0.81µs ± 2% -23.63% (p=0.000 n=19+20) 2M-48 2.07ms ± 0% 1.61ms ± 1% -22.31% (p=0.000 n=20+20) Write64-48 79.3ns ± 0% 58.0ns ± 1% -26.89% (p=0.000 n=20+19) Write1K-48 1.02µs ± 0% 0.79µs ± 1% -22.91% (p=0.000 n=19+19) Write2M-48 2.07ms ± 0% 1.61ms ± 2% -22.33% (p=0.000 n=17+20) name old speed new speed delta 64-48 508MB/s ± 0% 797MB/s ± 1% +56.95% (p=0.000 n=16+20) 1K-48 960MB/s ± 0% 1257MB/s ± 2% +30.94% (p=0.000 n=18+20) 2M-48 1.01GB/s ± 0% 1.30GB/s ± 1% +28.73% (p=0.000 n=20+20) Write64-48 807MB/s ± 0% 1104MB/s ± 1% +36.78% (p=0.000 n=18+19) Write1K-48 1.00GB/s ± 0% 1.30GB/s ± 1% +29.71% (p=0.000 n=18+19) Write2M-48 1.01GB/s ± 0% 1.31GB/s ± 2% +28.77% (p=0.000 n=17+20) The assembly is still 50-90% faster on the Xeon, 30-60% on the MacBook. The Go code does not use all the arithmetic tricks the assembly does, and it does not have access to the three operand wide shift instruction. name old time/op new time/op delta 64-48 80.3ns ± 1% 54.2ns ± 0% -32.50% (p=0.000 n=20+17) 1K-48 815ns ± 2% 446ns ± 1% -45.27% (p=0.000 n=20+20) 2M-48 1.61ms ± 1% 0.86ms ± 0% -46.54% (p=0.000 n=20+17) Write64-48 58.0ns ± 1% 34.0ns ± 0% -41.34% (p=0.000 n=19+20) Write1K-48 790ns ± 1% 427ns ± 0% -45.92% (p=0.000 n=19+17) Write2M-48 1.61ms ± 2% 0.86ms ± 0% -46.51% (p=0.000 n=20+20) name old speed new speed delta 64-48 797MB/s ± 1% 1180MB/s ± 0% +48.09% (p=0.000 n=20+19) 1K-48 1.26GB/s ± 2% 2.30GB/s ± 1% +82.71% (p=0.000 n=20+20) 2M-48 1.30GB/s ± 1% 2.44GB/s ± 0% +87.04% (p=0.000 n=20+17) Write64-48 1.10GB/s ± 1% 1.88GB/s ± 0% +70.52% (p=0.000 n=19+18) Write1K-48 1.30GB/s ± 1% 2.40GB/s ± 0% +84.84% (p=0.000 n=19+18) Write2M-48 1.31GB/s ± 2% 2.44GB/s ± 0% +86.93% (p=0.000 n=20+20) Hopefully this will also avoid the need for an arm64 implementation. Since now the Go and the amd64/ppc64le assembly use the same limb schedule, drop the assembly initialize and finalize implementations, and make the wrapper code match. It comes with a minor slowdown. name old time/op new time/op delta 64-48 50.3ns ± 0% 54.2ns ± 0% +7.73% (p=0.000 n=20+17) 1K-48 441ns ± 0% 446ns ± 1% +1.10% (p=0.000 n=19+20) 2M-48 860µs ± 0% 859µs ± 0% ~ (p=0.178 n=19+17) Write64-48 34.0ns ± 0% 34.0ns ± 0% ~ (all equal) Write1K-48 424ns ± 0% 427ns ± 0% +0.71% (p=0.000 n=17+17) Write2M-48 860µs ± 0% 859µs ± 0% -0.04% (p=0.000 n=19+20) name old speed new speed delta 64-48 1.27GB/s ± 0% 1.18GB/s ± 0% -7.20% (p=0.000 n=20+19) 1K-48 2.32GB/s ± 0% 2.30GB/s ± 1% -1.07% (p=0.000 n=18+20) 2M-48 2.44GB/s ± 0% 2.44GB/s ± 0% ~ (p=0.173 n=19+17) Write64-48 1.88GB/s ± 0% 1.88GB/s ± 0% +0.04% (p=0.000 n=19+18) Write1K-48 2.41GB/s ± 0% 2.40GB/s ± 0% -0.67% (p=0.000 n=19+18) Write2M-48 2.44GB/s ± 0% 2.44GB/s ± 0% +0.04% (p=0.000 n=19+20) Since poly1305/sum_generic.go was almost entirely rewritten, it's probably best reviewed on gitiles. This is the implementation published at https://blog.filippo.io/a-literate-go-implementation-of-poly1305/ Updates #31470 Change-Id: I74f9011d3ee317a43b05ae7f05d96081d08bffd3 Reviewed-on: https://go-review.googlesource.com/c/crypto/+/169037 Reviewed-by: Katie Hockman <katie@golang.org>
182 lines
3.3 KiB
ArmAsm
182 lines
3.3 KiB
ArmAsm
// Copyright 2019 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build ppc64le,!gccgo,!appengine
|
|
|
|
#include "textflag.h"
|
|
|
|
// This was ported from the amd64 implementation.
|
|
|
|
#define POLY1305_ADD(msg, h0, h1, h2, t0, t1, t2) \
|
|
MOVD (msg), t0; \
|
|
MOVD 8(msg), t1; \
|
|
MOVD $1, t2; \
|
|
ADDC t0, h0, h0; \
|
|
ADDE t1, h1, h1; \
|
|
ADDE t2, h2; \
|
|
ADD $16, msg
|
|
|
|
#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3, t4, t5) \
|
|
MULLD r0, h0, t0; \
|
|
MULLD r0, h1, t4; \
|
|
MULHDU r0, h0, t1; \
|
|
MULHDU r0, h1, t5; \
|
|
ADDC t4, t1, t1; \
|
|
MULLD r0, h2, t2; \
|
|
ADDZE t5; \
|
|
MULHDU r1, h0, t4; \
|
|
MULLD r1, h0, h0; \
|
|
ADD t5, t2, t2; \
|
|
ADDC h0, t1, t1; \
|
|
MULLD h2, r1, t3; \
|
|
ADDZE t4, h0; \
|
|
MULHDU r1, h1, t5; \
|
|
MULLD r1, h1, t4; \
|
|
ADDC t4, t2, t2; \
|
|
ADDE t5, t3, t3; \
|
|
ADDC h0, t2, t2; \
|
|
MOVD $-4, t4; \
|
|
MOVD t0, h0; \
|
|
MOVD t1, h1; \
|
|
ADDZE t3; \
|
|
ANDCC $3, t2, h2; \
|
|
AND t2, t4, t0; \
|
|
ADDC t0, h0, h0; \
|
|
ADDE t3, h1, h1; \
|
|
SLD $62, t3, t4; \
|
|
SRD $2, t2; \
|
|
ADDZE h2; \
|
|
OR t4, t2, t2; \
|
|
SRD $2, t3; \
|
|
ADDC t2, h0, h0; \
|
|
ADDE t3, h1, h1; \
|
|
ADDZE h2
|
|
|
|
DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
|
|
DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
|
|
GLOBL ·poly1305Mask<>(SB), RODATA, $16
|
|
|
|
// func update(state *[7]uint64, msg []byte)
|
|
TEXT ·update(SB), $0-32
|
|
MOVD state+0(FP), R3
|
|
MOVD msg_base+8(FP), R4
|
|
MOVD msg_len+16(FP), R5
|
|
|
|
MOVD 0(R3), R8 // h0
|
|
MOVD 8(R3), R9 // h1
|
|
MOVD 16(R3), R10 // h2
|
|
MOVD 24(R3), R11 // r0
|
|
MOVD 32(R3), R12 // r1
|
|
|
|
CMP R5, $16
|
|
BLT bytes_between_0_and_15
|
|
|
|
loop:
|
|
POLY1305_ADD(R4, R8, R9, R10, R20, R21, R22)
|
|
|
|
multiply:
|
|
POLY1305_MUL(R8, R9, R10, R11, R12, R16, R17, R18, R14, R20, R21)
|
|
ADD $-16, R5
|
|
CMP R5, $16
|
|
BGE loop
|
|
|
|
bytes_between_0_and_15:
|
|
CMP $0, R5
|
|
BEQ done
|
|
MOVD $0, R16 // h0
|
|
MOVD $0, R17 // h1
|
|
|
|
flush_buffer:
|
|
CMP R5, $8
|
|
BLE just1
|
|
|
|
MOVD $8, R21
|
|
SUB R21, R5, R21
|
|
|
|
// Greater than 8 -- load the rightmost remaining bytes in msg
|
|
// and put into R17 (h1)
|
|
MOVD (R4)(R21), R17
|
|
MOVD $16, R22
|
|
|
|
// Find the offset to those bytes
|
|
SUB R5, R22, R22
|
|
SLD $3, R22
|
|
|
|
// Shift to get only the bytes in msg
|
|
SRD R22, R17, R17
|
|
|
|
// Put 1 at high end
|
|
MOVD $1, R23
|
|
SLD $3, R21
|
|
SLD R21, R23, R23
|
|
OR R23, R17, R17
|
|
|
|
// Remainder is 8
|
|
MOVD $8, R5
|
|
|
|
just1:
|
|
CMP R5, $8
|
|
BLT less8
|
|
|
|
// Exactly 8
|
|
MOVD (R4), R16
|
|
|
|
CMP $0, R17
|
|
|
|
// Check if we've already set R17; if not
|
|
// set 1 to indicate end of msg.
|
|
BNE carry
|
|
MOVD $1, R17
|
|
BR carry
|
|
|
|
less8:
|
|
MOVD $0, R16 // h0
|
|
MOVD $0, R22 // shift count
|
|
CMP R5, $4
|
|
BLT less4
|
|
MOVWZ (R4), R16
|
|
ADD $4, R4
|
|
ADD $-4, R5
|
|
MOVD $32, R22
|
|
|
|
less4:
|
|
CMP R5, $2
|
|
BLT less2
|
|
MOVHZ (R4), R21
|
|
SLD R22, R21, R21
|
|
OR R16, R21, R16
|
|
ADD $16, R22
|
|
ADD $-2, R5
|
|
ADD $2, R4
|
|
|
|
less2:
|
|
CMP $0, R5
|
|
BEQ insert1
|
|
MOVBZ (R4), R21
|
|
SLD R22, R21, R21
|
|
OR R16, R21, R16
|
|
ADD $8, R22
|
|
|
|
insert1:
|
|
// Insert 1 at end of msg
|
|
MOVD $1, R21
|
|
SLD R22, R21, R21
|
|
OR R16, R21, R16
|
|
|
|
carry:
|
|
// Add new values to h0, h1, h2
|
|
ADDC R16, R8
|
|
ADDE R17, R9
|
|
ADDE $0, R10
|
|
MOVD $16, R5
|
|
ADD R5, R4
|
|
BR multiply
|
|
|
|
done:
|
|
// Save h0, h1, h2 in state
|
|
MOVD R8, 0(R3)
|
|
MOVD R9, 8(R3)
|
|
MOVD R10, 16(R3)
|
|
RET
|