From 1f7c531b64a19f04f56ccc318130688c5ba0aa51 Mon Sep 17 00:00:00 2001 From: Xiaolin Zhao Date: Fri, 21 Mar 2025 10:54:57 +0800 Subject: [PATCH] blake2b: add loong64 SIMD implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmark results on Loongson-3C6000 @ 2.20GHz: goos: linux goarch: loong64 pkg: golang.org/x/crypto/blake2b cpu: Loongson-3C6000/S @ 2200.00MHz | old.bench | new.bench | | sec/op | sec/op vs base | Write128 350.0n ± 0% 178.3n ± 0% -49.06% (p=0.000 n=10) Write1K 2.687µ ± 0% 1.286µ ± 0% -52.14% (p=0.000 n=10) Sum128 362.6n ± 0% 183.7n ± 0% -49.34% (p=0.000 n=10) Sum1K 2.788µ ± 0% 1.295µ ± 0% -53.54% (p=0.000 n=10) geomean 987.4n 483.3n -51.06% | old.bench | new.bench | | B/s | B/s vs base | Write128 348.8Mi ± 0% 684.8Mi ± 0% +96.31% (p=0.000 n=10) Write1K 363.5Mi ± 0% 759.6Mi ± 0% +108.99% (p=0.000 n=10) Sum128 336.6Mi ± 0% 664.4Mi ± 0% +97.37% (p=0.000 n=10) Sum1K 350.3Mi ± 0% 754.0Mi ± 0% +115.23% (p=0.000 n=10) geomean 349.7Mi 714.5Mi +104.32% Change-Id: I181a0284c5913cca34f074466f7d83439554e450 Reviewed-on: https://go-review.googlesource.com/c/crypto/+/659775 LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: abner chenc Reviewed-by: Filippo Valsorda Reviewed-by: Junyang Shao Reviewed-by: Cherry Mui --- blake2b/blake2b_loong64.go | 20 +++ blake2b/blake2b_loong64.s | 252 +++++++++++++++++++++++++++++++++++++ blake2b/blake2b_ref.go | 2 +- 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 blake2b/blake2b_loong64.go create mode 100644 blake2b/blake2b_loong64.s diff --git a/blake2b/blake2b_loong64.go b/blake2b/blake2b_loong64.go new file mode 100644 index 0000000..5e979eb --- /dev/null +++ b/blake2b/blake2b_loong64.go @@ -0,0 +1,20 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && gc && !purego + +package blake2b + +import "golang.org/x/sys/cpu" + +//go:noescape +func hashBlocksVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) + +func hashBlocks(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) { + if cpu.Loong64.HasLSX { + hashBlocksVX(h, c, flag, blocks) + } else { + hashBlocksGeneric(h, c, flag, blocks) + } +} diff --git a/blake2b/blake2b_loong64.s b/blake2b/blake2b_loong64.s new file mode 100644 index 0000000..be9c436 --- /dev/null +++ b/blake2b/blake2b_loong64.s @@ -0,0 +1,252 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && gc && !purego + +#include "textflag.h" + +DATA ·iv0<>+0(SB)/8, $0x6a09e667f3bcc908 +DATA ·iv0<>+8(SB)/8, $0xbb67ae8584caa73b +GLOBL ·iv0<>(SB), RODATA|NOPTR, $16 + +DATA ·iv1<>+0(SB)/8, $0x3c6ef372fe94f82b +DATA ·iv1<>+8(SB)/8, $0xa54ff53a5f1d36f1 +GLOBL ·iv1<>(SB), RODATA|NOPTR, $16 + +DATA ·iv2<>+0(SB)/8, $0x510e527fade682d1 +DATA ·iv2<>+8(SB)/8, $0x9b05688c2b3e6c1f +GLOBL ·iv2<>(SB), RODATA|NOPTR, $16 + +DATA ·iv3<>+0(SB)/8, $0x1f83d9abfb41bd6b +DATA ·iv3<>+8(SB)/8, $0x5be0cd19137e2179 +GLOBL ·iv3<>(SB), RODATA|NOPTR, $16 + +#define SHUFFLE_1 \ + VADDV V2, V13, V12; \ // V12 = V2 + VADDV V6, V13, V16; \ // V16 = V6 + VADDV V4, V13, V14; \ // V14 = V4 + VADDV V5, V13, V4; \ // V4 = V5 + VADDV V14, V13, V5; \ // V5 = V4 + VSHUF4IV $9, V3, V2; \ + VSHUF4IV $9, V12, V3; \ + VSHUF4IV $3, V7, V6; \ + VSHUF4IV $3, V16, V7; \ + +#define SHUFFLE_2 \ + VADDV V2, V13, V12; \ // V12 = V2 + VADDV V6, V13, V16; \ // V16 = V6 + VADDV V4, V13, V14; \ // V14 = V4 + VADDV V5, V13, V4; \ // V4 = V5 + VADDV V14, V13, V5; \ // V5 = V4 + VSHUF4IV $3, V3, V2; \ + VSHUF4IV $3, V12, V3; \ + VSHUF4IV $9, V7, V6; \ + VSHUF4IV $9, V16, V7; \ + +#define LOAD_M(a, b, c, d, e, f, g, h) \ + VMOVQ a, V8.V[0]; \ + VMOVQ b, V8.V[1]; \ + VMOVQ c, V9.V[0]; \ + VMOVQ d, V9.V[1]; \ + VMOVQ e, V10.V[0]; \ + VMOVQ f, V10.V[1]; \ + VMOVQ g, V11.V[0]; \ + VMOVQ h, V11.V[1]; \ + +#define ROUND_0 \ + VADDV V0, V8, V0; \ + VADDV V1, V9, V1; \ + VADDV V0, V2, V0; \ + VADDV V1, V3, V1; \ + VXORV V6, V0, V6; \ + VXORV V7, V1, V7; \ + VROTRV $32, V6, V6; \ + VROTRV $32, V7, V7; \ + VADDV V4, V6, V4; \ + VADDV V5, V7, V5; \ + VXORV V2, V4, V2; \ + VXORV V3, V5, V3; \ + VROTRV $24, V2, V2; \ + VROTRV $24, V3, V3; \ + VADDV V0, V10, V0; \ + VADDV V1, V11, V1; \ + VADDV V0, V2, V0; \ + VADDV V1, V3, V1; \ + VXORV V6, V0, V6; \ + VXORV V7, V1, V7; \ + VROTRV $16, V6, V6; \ + VROTRV $16, V7, V7; \ + VADDV V4, V6, V4; \ + VADDV V5, V7, V5; \ + VXORV V2, V4, V2; \ + VXORV V3, V5, V3; \ + VROTRV $63, V2, V2; \ + VROTRV $63, V3, V3; \ + +#define ROUND_8 ROUND_0 + +// func hashBlocksVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) +TEXT ·hashBlocksVX(SB), NOSPLIT, $0-48 + MOVV h+0(FP), R4 + MOVV c+8(FP), R5 + MOVV flag+16(FP), R6 + MOVV blocks_base+24(FP), R7 + MOVV blocks_len+32(FP), R8 + MOVV (R5), R9 // c0 + MOVV 8(R5), R10 // c1 + +loop: + ADDV $0x80, R9 + SGTU $0x80, R9, R11 + ADDV R10, R11, R10 + VMOVQ R9, V8.V[0] + VMOVQ R10, V8.V[1] + + MOVV $·iv0<>(SB), R11 + MOVV $·iv1<>(SB), R12 + MOVV $·iv2<>(SB), R13 + MOVV $·iv3<>(SB), R14 + VMOVQ (R13), V6 + VMOVQ (R11), V4 + VMOVQ (R12), V5 + MOVV (R14), R15 // v14 + MOVV 8(R14), R16 + VXORV V6, V8, V6 + XOR R15, R6, R15 + VMOVQ R16, V7.V[1] + VMOVQ R15, V7.V[0] + + VMOVQ (R4), V0 + VMOVQ 16(R4), V1 + VMOVQ 32(R4), V2 + VMOVQ 48(R4), V3 + + MOVV 0(R7), R11 + MOVV 8(R7), R12 + MOVV 16(R7), R13 + MOVV 24(R7), R14 + MOVV 32(R7), R15 + MOVV 40(R7), R16 + MOVV 48(R7), R17 + MOVV 56(R7), R18 + MOVV 64(R7), R19 + MOVV 72(R7), R24 + MOVV 80(R7), R25 + MOVV 88(R7), R26 + MOVV 96(R7), R27 + MOVV 104(R7), R28 + MOVV 112(R7), R29 + MOVV 120(R7), R30 + + VXORV V13, V13, V13 // V13=0 + + LOAD_M(R11, R13, R15, R17, R12, R14, R16, R18) + ROUND_0 + SHUFFLE_1 + LOAD_M(R19, R25, R27, R29, R24, R26, R28, R30) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R29, R15, R24, R28, R25, R19, R30, R17) + ROUND_0 + SHUFFLE_1 + LOAD_M(R12, R11, R26, R16, R27, R13, R18, R14) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R26, R27, R16, R30, R19, R11, R13, R28) + ROUND_0 + SHUFFLE_1 + LOAD_M(R25, R14, R18, R24, R29, R17, R12, R15) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R18, R14, R28, R26, R24, R12, R27, R29) + ROUND_0 + SHUFFLE_1 + LOAD_M(R13, R16, R15, R30, R17, R25, R11, R19) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R24, R16, R13, R25, R11, R18, R15, R30) + ROUND_0 + SHUFFLE_1 + LOAD_M(R29, R26, R17, R14, R12, R27, R19, R28) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R13, R17, R11, R19, R27, R25, R26, R14) + ROUND_0 + SHUFFLE_1 + LOAD_M(R15, R18, R30, R12, R28, R16, R29, R24) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R27, R12, R29, R15, R16, R30, R28, R25) + ROUND_0 + SHUFFLE_1 + LOAD_M(R11, R17, R24, R19, R18, R14, R13, R26) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R28, R18, R27, R14, R26, R29, R12, R24) + ROUND_0 + SHUFFLE_1 + LOAD_M(R16, R30, R19, R13, R11, R15, R17, R25) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R17, R29, R26, R11, R30, R24, R14, R19) + ROUND_0 + SHUFFLE_1 + LOAD_M(R27, R28, R12, R25, R13, R18, R15, R16) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R25, R19, R18, R12, R13, R15, R17, R16) + ROUND_0 + SHUFFLE_1 + LOAD_M(R30, R24, R14, R28, R26, R29, R27, R11) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R11, R13, R15, R17, R12, R14, R16, R18) + ROUND_0 + SHUFFLE_1 + LOAD_M(R19, R25, R27, R29, R24, R26, R28, R30) + ROUND_8 + SHUFFLE_2 + + LOAD_M(R29, R15, R24, R28, R25, R19, R30, R17) + ROUND_0 + SHUFFLE_1 + LOAD_M(R12, R11, R26, R16, R27, R13, R18, R14) + ROUND_8 + SHUFFLE_2 + + VMOVQ (R4), V8 + VMOVQ 16(R4), V9 + VMOVQ 32(R4), V10 + VMOVQ 48(R4), V11 + VXORV V0, V4, V0 + VXORV V1, V5, V1 + VXORV V2, V6, V2 + VXORV V3, V7, V3 + VXORV V0, V8, V0 + VXORV V1, V9, V1 + VXORV V2, V10, V2 + VXORV V3, V11, V3 + VMOVQ V0, (R4) + VMOVQ V1, 16(R4) + VMOVQ V2, 32(R4) + VMOVQ V3, 48(R4) + + SUBV $128, R8 + ADDV $128, R7 + BNE R8, R0, loop + + MOVV R9, (R5) + MOVV R10, 8(R5) + + RET diff --git a/blake2b/blake2b_ref.go b/blake2b/blake2b_ref.go index 6e28668..70852a0 100644 --- a/blake2b/blake2b_ref.go +++ b/blake2b/blake2b_ref.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 || purego || !gc +//go:build (!amd64 && !loong64) || purego || !gc package blake2b