Filippo Valsorda
2dbfe9001f
The new code is meant to be readable without external references for Poly1305, and explains the field logic. The generic code is now 30-50% faster on a Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz, and even better on a 3.1 GHz i7 MacBook. name old time/op new time/op delta 64-48 126ns ± 0% 80ns ± 1% -36.24% (p=0.000 n=16+20) 1K-48 1.07µs ± 0% 0.81µs ± 2% -23.63% (p=0.000 n=19+20) 2M-48 2.07ms ± 0% 1.61ms ± 1% -22.31% (p=0.000 n=20+20) Write64-48 79.3ns ± 0% 58.0ns ± 1% -26.89% (p=0.000 n=20+19) Write1K-48 1.02µs ± 0% 0.79µs ± 1% -22.91% (p=0.000 n=19+19) Write2M-48 2.07ms ± 0% 1.61ms ± 2% -22.33% (p=0.000 n=17+20) name old speed new speed delta 64-48 508MB/s ± 0% 797MB/s ± 1% +56.95% (p=0.000 n=16+20) 1K-48 960MB/s ± 0% 1257MB/s ± 2% +30.94% (p=0.000 n=18+20) 2M-48 1.01GB/s ± 0% 1.30GB/s ± 1% +28.73% (p=0.000 n=20+20) Write64-48 807MB/s ± 0% 1104MB/s ± 1% +36.78% (p=0.000 n=18+19) Write1K-48 1.00GB/s ± 0% 1.30GB/s ± 1% +29.71% (p=0.000 n=18+19) Write2M-48 1.01GB/s ± 0% 1.31GB/s ± 2% +28.77% (p=0.000 n=17+20) The assembly is still 50-90% faster on the Xeon, 30-60% on the MacBook. The Go code does not use all the arithmetic tricks the assembly does, and it does not have access to the three operand wide shift instruction. name old time/op new time/op delta 64-48 80.3ns ± 1% 54.2ns ± 0% -32.50% (p=0.000 n=20+17) 1K-48 815ns ± 2% 446ns ± 1% -45.27% (p=0.000 n=20+20) 2M-48 1.61ms ± 1% 0.86ms ± 0% -46.54% (p=0.000 n=20+17) Write64-48 58.0ns ± 1% 34.0ns ± 0% -41.34% (p=0.000 n=19+20) Write1K-48 790ns ± 1% 427ns ± 0% -45.92% (p=0.000 n=19+17) Write2M-48 1.61ms ± 2% 0.86ms ± 0% -46.51% (p=0.000 n=20+20) name old speed new speed delta 64-48 797MB/s ± 1% 1180MB/s ± 0% +48.09% (p=0.000 n=20+19) 1K-48 1.26GB/s ± 2% 2.30GB/s ± 1% +82.71% (p=0.000 n=20+20) 2M-48 1.30GB/s ± 1% 2.44GB/s ± 0% +87.04% (p=0.000 n=20+17) Write64-48 1.10GB/s ± 1% 1.88GB/s ± 0% +70.52% (p=0.000 n=19+18) Write1K-48 1.30GB/s ± 1% 2.40GB/s ± 0% +84.84% (p=0.000 n=19+18) Write2M-48 1.31GB/s ± 2% 2.44GB/s ± 0% +86.93% (p=0.000 n=20+20) Hopefully this will also avoid the need for an arm64 implementation. Since now the Go and the amd64/ppc64le assembly use the same limb schedule, drop the assembly initialize and finalize implementations, and make the wrapper code match. It comes with a minor slowdown. name old time/op new time/op delta 64-48 50.3ns ± 0% 54.2ns ± 0% +7.73% (p=0.000 n=20+17) 1K-48 441ns ± 0% 446ns ± 1% +1.10% (p=0.000 n=19+20) 2M-48 860µs ± 0% 859µs ± 0% ~ (p=0.178 n=19+17) Write64-48 34.0ns ± 0% 34.0ns ± 0% ~ (all equal) Write1K-48 424ns ± 0% 427ns ± 0% +0.71% (p=0.000 n=17+17) Write2M-48 860µs ± 0% 859µs ± 0% -0.04% (p=0.000 n=19+20) name old speed new speed delta 64-48 1.27GB/s ± 0% 1.18GB/s ± 0% -7.20% (p=0.000 n=20+19) 1K-48 2.32GB/s ± 0% 2.30GB/s ± 1% -1.07% (p=0.000 n=18+20) 2M-48 2.44GB/s ± 0% 2.44GB/s ± 0% ~ (p=0.173 n=19+17) Write64-48 1.88GB/s ± 0% 1.88GB/s ± 0% +0.04% (p=0.000 n=19+18) Write1K-48 2.41GB/s ± 0% 2.40GB/s ± 0% -0.67% (p=0.000 n=19+18) Write2M-48 2.44GB/s ± 0% 2.44GB/s ± 0% +0.04% (p=0.000 n=19+20) Since poly1305/sum_generic.go was almost entirely rewritten, it's probably best reviewed on gitiles. This is the implementation published at https://blog.filippo.io/a-literate-go-implementation-of-poly1305/ Updates #31470 Change-Id: I74f9011d3ee317a43b05ae7f05d96081d08bffd3 Reviewed-on: https://go-review.googlesource.com/c/crypto/+/169037 Reviewed-by: Katie Hockman <katie@golang.org>
109 lines
2.5 KiB
ArmAsm
109 lines
2.5 KiB
ArmAsm
// Copyright 2012 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build amd64,!gccgo,!appengine
|
|
|
|
#include "textflag.h"
|
|
|
|
#define POLY1305_ADD(msg, h0, h1, h2) \
|
|
ADDQ 0(msg), h0; \
|
|
ADCQ 8(msg), h1; \
|
|
ADCQ $1, h2; \
|
|
LEAQ 16(msg), msg
|
|
|
|
#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \
|
|
MOVQ r0, AX; \
|
|
MULQ h0; \
|
|
MOVQ AX, t0; \
|
|
MOVQ DX, t1; \
|
|
MOVQ r0, AX; \
|
|
MULQ h1; \
|
|
ADDQ AX, t1; \
|
|
ADCQ $0, DX; \
|
|
MOVQ r0, t2; \
|
|
IMULQ h2, t2; \
|
|
ADDQ DX, t2; \
|
|
\
|
|
MOVQ r1, AX; \
|
|
MULQ h0; \
|
|
ADDQ AX, t1; \
|
|
ADCQ $0, DX; \
|
|
MOVQ DX, h0; \
|
|
MOVQ r1, t3; \
|
|
IMULQ h2, t3; \
|
|
MOVQ r1, AX; \
|
|
MULQ h1; \
|
|
ADDQ AX, t2; \
|
|
ADCQ DX, t3; \
|
|
ADDQ h0, t2; \
|
|
ADCQ $0, t3; \
|
|
\
|
|
MOVQ t0, h0; \
|
|
MOVQ t1, h1; \
|
|
MOVQ t2, h2; \
|
|
ANDQ $3, h2; \
|
|
MOVQ t2, t0; \
|
|
ANDQ $0xFFFFFFFFFFFFFFFC, t0; \
|
|
ADDQ t0, h0; \
|
|
ADCQ t3, h1; \
|
|
ADCQ $0, h2; \
|
|
SHRQ $2, t3, t2; \
|
|
SHRQ $2, t3; \
|
|
ADDQ t2, h0; \
|
|
ADCQ t3, h1; \
|
|
ADCQ $0, h2
|
|
|
|
// func update(state *[7]uint64, msg []byte)
|
|
TEXT ·update(SB), $0-32
|
|
MOVQ state+0(FP), DI
|
|
MOVQ msg_base+8(FP), SI
|
|
MOVQ msg_len+16(FP), R15
|
|
|
|
MOVQ 0(DI), R8 // h0
|
|
MOVQ 8(DI), R9 // h1
|
|
MOVQ 16(DI), R10 // h2
|
|
MOVQ 24(DI), R11 // r0
|
|
MOVQ 32(DI), R12 // r1
|
|
|
|
CMPQ R15, $16
|
|
JB bytes_between_0_and_15
|
|
|
|
loop:
|
|
POLY1305_ADD(SI, R8, R9, R10)
|
|
|
|
multiply:
|
|
POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14)
|
|
SUBQ $16, R15
|
|
CMPQ R15, $16
|
|
JAE loop
|
|
|
|
bytes_between_0_and_15:
|
|
TESTQ R15, R15
|
|
JZ done
|
|
MOVQ $1, BX
|
|
XORQ CX, CX
|
|
XORQ R13, R13
|
|
ADDQ R15, SI
|
|
|
|
flush_buffer:
|
|
SHLQ $8, BX, CX
|
|
SHLQ $8, BX
|
|
MOVB -1(SI), R13
|
|
XORQ R13, BX
|
|
DECQ SI
|
|
DECQ R15
|
|
JNZ flush_buffer
|
|
|
|
ADDQ BX, R8
|
|
ADCQ CX, R9
|
|
ADCQ $0, R10
|
|
MOVQ $16, R15
|
|
JMP multiply
|
|
|
|
done:
|
|
MOVQ R8, 0(DI)
|
|
MOVQ R9, 8(DI)
|
|
MOVQ R10, 16(DI)
|
|
RET
|