blake2b: add loong64 SIMD implementation
Benchmark results on Loongson-3C6000 @ 2.20GHz:
goos: linux
goarch: loong64
pkg: golang.org/x/crypto/blake2b
cpu: Loongson-3C6000/S @ 2200.00MHz
| old.bench | new.bench |
| sec/op | sec/op vs base |
Write128 350.0n ± 0% 178.3n ± 0% -49.06% (p=0.000 n=10)
Write1K 2.687µ ± 0% 1.286µ ± 0% -52.14% (p=0.000 n=10)
Sum128 362.6n ± 0% 183.7n ± 0% -49.34% (p=0.000 n=10)
Sum1K 2.788µ ± 0% 1.295µ ± 0% -53.54% (p=0.000 n=10)
geomean 987.4n 483.3n -51.06%
| old.bench | new.bench |
| B/s | B/s vs base |
Write128 348.8Mi ± 0% 684.8Mi ± 0% +96.31% (p=0.000 n=10)
Write1K 363.5Mi ± 0% 759.6Mi ± 0% +108.99% (p=0.000 n=10)
Sum128 336.6Mi ± 0% 664.4Mi ± 0% +97.37% (p=0.000 n=10)
Sum1K 350.3Mi ± 0% 754.0Mi ± 0% +115.23% (p=0.000 n=10)
geomean 349.7Mi 714.5Mi +104.32%
Change-Id: I181a0284c5913cca34f074466f7d83439554e450
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/659775
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: abner chenc <chenguoqi@loongson.cn>
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Reviewed-by: Junyang Shao <shaojunyang@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
This commit is contained in:
Xiaolin Zhao
committed by
abner chenc
abner chenc
parent
234d0b5a5f
commit
1f7c531b64
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build loong64 && gc && !purego
|
||||
|
||||
package blake2b
|
||||
|
||||
import "golang.org/x/sys/cpu"
|
||||
|
||||
//go:noescape
|
||||
func hashBlocksVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
|
||||
|
||||
func hashBlocks(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) {
|
||||
if cpu.Loong64.HasLSX {
|
||||
hashBlocksVX(h, c, flag, blocks)
|
||||
} else {
|
||||
hashBlocksGeneric(h, c, flag, blocks)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// Copyright 2026 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build loong64 && gc && !purego
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
DATA ·iv0<>+0(SB)/8, $0x6a09e667f3bcc908
|
||||
DATA ·iv0<>+8(SB)/8, $0xbb67ae8584caa73b
|
||||
GLOBL ·iv0<>(SB), RODATA|NOPTR, $16
|
||||
|
||||
DATA ·iv1<>+0(SB)/8, $0x3c6ef372fe94f82b
|
||||
DATA ·iv1<>+8(SB)/8, $0xa54ff53a5f1d36f1
|
||||
GLOBL ·iv1<>(SB), RODATA|NOPTR, $16
|
||||
|
||||
DATA ·iv2<>+0(SB)/8, $0x510e527fade682d1
|
||||
DATA ·iv2<>+8(SB)/8, $0x9b05688c2b3e6c1f
|
||||
GLOBL ·iv2<>(SB), RODATA|NOPTR, $16
|
||||
|
||||
DATA ·iv3<>+0(SB)/8, $0x1f83d9abfb41bd6b
|
||||
DATA ·iv3<>+8(SB)/8, $0x5be0cd19137e2179
|
||||
GLOBL ·iv3<>(SB), RODATA|NOPTR, $16
|
||||
|
||||
#define SHUFFLE_1 \
|
||||
VADDV V2, V13, V12; \ // V12 = V2
|
||||
VADDV V6, V13, V16; \ // V16 = V6
|
||||
VADDV V4, V13, V14; \ // V14 = V4
|
||||
VADDV V5, V13, V4; \ // V4 = V5
|
||||
VADDV V14, V13, V5; \ // V5 = V4
|
||||
VSHUF4IV $9, V3, V2; \
|
||||
VSHUF4IV $9, V12, V3; \
|
||||
VSHUF4IV $3, V7, V6; \
|
||||
VSHUF4IV $3, V16, V7; \
|
||||
|
||||
#define SHUFFLE_2 \
|
||||
VADDV V2, V13, V12; \ // V12 = V2
|
||||
VADDV V6, V13, V16; \ // V16 = V6
|
||||
VADDV V4, V13, V14; \ // V14 = V4
|
||||
VADDV V5, V13, V4; \ // V4 = V5
|
||||
VADDV V14, V13, V5; \ // V5 = V4
|
||||
VSHUF4IV $3, V3, V2; \
|
||||
VSHUF4IV $3, V12, V3; \
|
||||
VSHUF4IV $9, V7, V6; \
|
||||
VSHUF4IV $9, V16, V7; \
|
||||
|
||||
#define LOAD_M(a, b, c, d, e, f, g, h) \
|
||||
VMOVQ a, V8.V[0]; \
|
||||
VMOVQ b, V8.V[1]; \
|
||||
VMOVQ c, V9.V[0]; \
|
||||
VMOVQ d, V9.V[1]; \
|
||||
VMOVQ e, V10.V[0]; \
|
||||
VMOVQ f, V10.V[1]; \
|
||||
VMOVQ g, V11.V[0]; \
|
||||
VMOVQ h, V11.V[1]; \
|
||||
|
||||
#define ROUND_0 \
|
||||
VADDV V0, V8, V0; \
|
||||
VADDV V1, V9, V1; \
|
||||
VADDV V0, V2, V0; \
|
||||
VADDV V1, V3, V1; \
|
||||
VXORV V6, V0, V6; \
|
||||
VXORV V7, V1, V7; \
|
||||
VROTRV $32, V6, V6; \
|
||||
VROTRV $32, V7, V7; \
|
||||
VADDV V4, V6, V4; \
|
||||
VADDV V5, V7, V5; \
|
||||
VXORV V2, V4, V2; \
|
||||
VXORV V3, V5, V3; \
|
||||
VROTRV $24, V2, V2; \
|
||||
VROTRV $24, V3, V3; \
|
||||
VADDV V0, V10, V0; \
|
||||
VADDV V1, V11, V1; \
|
||||
VADDV V0, V2, V0; \
|
||||
VADDV V1, V3, V1; \
|
||||
VXORV V6, V0, V6; \
|
||||
VXORV V7, V1, V7; \
|
||||
VROTRV $16, V6, V6; \
|
||||
VROTRV $16, V7, V7; \
|
||||
VADDV V4, V6, V4; \
|
||||
VADDV V5, V7, V5; \
|
||||
VXORV V2, V4, V2; \
|
||||
VXORV V3, V5, V3; \
|
||||
VROTRV $63, V2, V2; \
|
||||
VROTRV $63, V3, V3; \
|
||||
|
||||
#define ROUND_8 ROUND_0
|
||||
|
||||
// func hashBlocksVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
|
||||
TEXT ·hashBlocksVX(SB), NOSPLIT, $0-48
|
||||
MOVV h+0(FP), R4
|
||||
MOVV c+8(FP), R5
|
||||
MOVV flag+16(FP), R6
|
||||
MOVV blocks_base+24(FP), R7
|
||||
MOVV blocks_len+32(FP), R8
|
||||
MOVV (R5), R9 // c0
|
||||
MOVV 8(R5), R10 // c1
|
||||
|
||||
loop:
|
||||
ADDV $0x80, R9
|
||||
SGTU $0x80, R9, R11
|
||||
ADDV R10, R11, R10
|
||||
VMOVQ R9, V8.V[0]
|
||||
VMOVQ R10, V8.V[1]
|
||||
|
||||
MOVV $·iv0<>(SB), R11
|
||||
MOVV $·iv1<>(SB), R12
|
||||
MOVV $·iv2<>(SB), R13
|
||||
MOVV $·iv3<>(SB), R14
|
||||
VMOVQ (R13), V6
|
||||
VMOVQ (R11), V4
|
||||
VMOVQ (R12), V5
|
||||
MOVV (R14), R15 // v14
|
||||
MOVV 8(R14), R16
|
||||
VXORV V6, V8, V6
|
||||
XOR R15, R6, R15
|
||||
VMOVQ R16, V7.V[1]
|
||||
VMOVQ R15, V7.V[0]
|
||||
|
||||
VMOVQ (R4), V0
|
||||
VMOVQ 16(R4), V1
|
||||
VMOVQ 32(R4), V2
|
||||
VMOVQ 48(R4), V3
|
||||
|
||||
MOVV 0(R7), R11
|
||||
MOVV 8(R7), R12
|
||||
MOVV 16(R7), R13
|
||||
MOVV 24(R7), R14
|
||||
MOVV 32(R7), R15
|
||||
MOVV 40(R7), R16
|
||||
MOVV 48(R7), R17
|
||||
MOVV 56(R7), R18
|
||||
MOVV 64(R7), R19
|
||||
MOVV 72(R7), R24
|
||||
MOVV 80(R7), R25
|
||||
MOVV 88(R7), R26
|
||||
MOVV 96(R7), R27
|
||||
MOVV 104(R7), R28
|
||||
MOVV 112(R7), R29
|
||||
MOVV 120(R7), R30
|
||||
|
||||
VXORV V13, V13, V13 // V13=0
|
||||
|
||||
LOAD_M(R11, R13, R15, R17, R12, R14, R16, R18)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R19, R25, R27, R29, R24, R26, R28, R30)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R29, R15, R24, R28, R25, R19, R30, R17)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R12, R11, R26, R16, R27, R13, R18, R14)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R26, R27, R16, R30, R19, R11, R13, R28)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R25, R14, R18, R24, R29, R17, R12, R15)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R18, R14, R28, R26, R24, R12, R27, R29)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R13, R16, R15, R30, R17, R25, R11, R19)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R24, R16, R13, R25, R11, R18, R15, R30)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R29, R26, R17, R14, R12, R27, R19, R28)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R13, R17, R11, R19, R27, R25, R26, R14)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R15, R18, R30, R12, R28, R16, R29, R24)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R27, R12, R29, R15, R16, R30, R28, R25)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R11, R17, R24, R19, R18, R14, R13, R26)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R28, R18, R27, R14, R26, R29, R12, R24)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R16, R30, R19, R13, R11, R15, R17, R25)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R17, R29, R26, R11, R30, R24, R14, R19)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R27, R28, R12, R25, R13, R18, R15, R16)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R25, R19, R18, R12, R13, R15, R17, R16)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R30, R24, R14, R28, R26, R29, R27, R11)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R11, R13, R15, R17, R12, R14, R16, R18)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R19, R25, R27, R29, R24, R26, R28, R30)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
LOAD_M(R29, R15, R24, R28, R25, R19, R30, R17)
|
||||
ROUND_0
|
||||
SHUFFLE_1
|
||||
LOAD_M(R12, R11, R26, R16, R27, R13, R18, R14)
|
||||
ROUND_8
|
||||
SHUFFLE_2
|
||||
|
||||
VMOVQ (R4), V8
|
||||
VMOVQ 16(R4), V9
|
||||
VMOVQ 32(R4), V10
|
||||
VMOVQ 48(R4), V11
|
||||
VXORV V0, V4, V0
|
||||
VXORV V1, V5, V1
|
||||
VXORV V2, V6, V2
|
||||
VXORV V3, V7, V3
|
||||
VXORV V0, V8, V0
|
||||
VXORV V1, V9, V1
|
||||
VXORV V2, V10, V2
|
||||
VXORV V3, V11, V3
|
||||
VMOVQ V0, (R4)
|
||||
VMOVQ V1, 16(R4)
|
||||
VMOVQ V2, 32(R4)
|
||||
VMOVQ V3, 48(R4)
|
||||
|
||||
SUBV $128, R8
|
||||
ADDV $128, R7
|
||||
BNE R8, R0, loop
|
||||
|
||||
MOVV R9, (R5)
|
||||
MOVV R10, 8(R5)
|
||||
|
||||
RET
|
||||
@@ -2,7 +2,7 @@
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !amd64 || purego || !gc
|
||||
//go:build (!amd64 && !loong64) || purego || !gc
|
||||
|
||||
package blake2b
|
||||
|
||||
|
||||
Reference in New Issue
Block a user