From 234d0b5a5f76653159d274d0e7750a994c39dc84 Mon Sep 17 00:00:00 2001 From: Xiaolin Zhao Date: Mon, 23 Jun 2025 11:28:29 +0800 Subject: [PATCH] chacha20: add loong64 SIMD implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an assembly implementation of ChaCha20 for Loong64 using LSX SIMD instructions. This significantly improves performance across all message sizes on LoongArch machines. Benchmark results on Loongson-3C6000 @ 2.20GHz: goos: linux goarch: loong64 pkg: golang.org/x/crypto/chacha20 cpu: Loongson-3C6000/S @ 2200.00MHz │ old.bench │ new.bench │ │ sec/op │ sec/op vs base │ ChaCha20/64 188.4n ± 0% 156.9n ± 0% -16.72% (p=0.000 n=10) ChaCha20/256 666.4n ± 0% 151.5n ± 0% -77.27% (p=0.000 n=10) ChaCha20/10x25 1037.0n ± 0% 578.9n ± 0% -44.18% (p=0.000 n=10) ChaCha20/4096 10.207µ ± 0% 1.955µ ± 0% -80.85% (p=0.000 n=10) ChaCha20/100x40 11.856µ ± 0% 5.785µ ± 0% -51.21% (p=0.000 n=10) ChaCha20/65536 162.99µ ± 0% 31.60µ ± 0% -80.61% (p=0.000 n=10) ChaCha20/1000x65 165.57µ ± 0% 41.93µ ± 0% -74.67% (p=0.000 n=10) geomean 6.369µ 2.141µ -66.38% │ old.bench │ new.bench │ │ B/s │ B/s vs base │ ChaCha20/64 323.9Mi ± 0% 389.0Mi ± 0% +20.10% (p=0.000 n=10) ChaCha20/256 366.4Mi ± 0% 1611.2Mi ± 0% +339.80% (p=0.000 n=10) ChaCha20/10x25 229.8Mi ± 0% 411.8Mi ± 0% +79.19% (p=0.000 n=10) ChaCha20/4096 382.7Mi ± 0% 1998.1Mi ± 0% +422.08% (p=0.000 n=10) ChaCha20/100x40 321.8Mi ± 0% 659.4Mi ± 0% +104.94% (p=0.000 n=10) ChaCha20/65536 383.5Mi ± 0% 1977.8Mi ± 0% +415.78% (p=0.000 n=10) ChaCha20/1000x65 374.4Mi ± 0% 1478.4Mi ± 0% +294.86% (p=0.000 n=10) geomean 335.9Mi 999.2Mi +197.50% Change-Id: I6b8d243819d0ab1a1d6540648ed9bd00ecad5a2e Reviewed-on: https://go-review.googlesource.com/c/crypto/+/683196 Reviewed-by: Junyang Shao LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: abner chenc Reviewed-by: Cherry Mui --- chacha20/chacha_loong64.go | 22 +++ chacha20/chacha_loong64.s | 342 +++++++++++++++++++++++++++++++++++++ chacha20/chacha_noasm.go | 2 +- 3 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 chacha20/chacha_loong64.go create mode 100644 chacha20/chacha_loong64.s diff --git a/chacha20/chacha_loong64.go b/chacha20/chacha_loong64.go new file mode 100644 index 0000000..9679f9e --- /dev/null +++ b/chacha20/chacha_loong64.go @@ -0,0 +1,22 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc && !purego + +package chacha20 + +import "golang.org/x/sys/cpu" + +const bufSize = 256 + +//go:noescape +func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32) + +func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) { + if cpu.Loong64.HasLSX { + xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter) + } else { + c.xorKeyStreamBlocksGeneric(dst, src) + } +} diff --git a/chacha20/chacha_loong64.s b/chacha20/chacha_loong64.s new file mode 100644 index 0000000..e377b43 --- /dev/null +++ b/chacha20/chacha_loong64.s @@ -0,0 +1,342 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// derived from chacha_arm64.s + +//go:build gc && !purego + +#include "textflag.h" + +DATA ·constants+0x00(SB)/4, $0x61707865 +DATA ·constants+0x04(SB)/4, $0x3320646e +DATA ·constants+0x08(SB)/4, $0x79622d32 +DATA ·constants+0x0c(SB)/4, $0x6b206574 +GLOBL ·constants(SB), NOPTR|RODATA, $32 + +DATA ·incRotMatrix+0x00(SB)/4, $0x00000000 +DATA ·incRotMatrix+0x04(SB)/4, $0x00000001 +DATA ·incRotMatrix+0x08(SB)/4, $0x00000002 +DATA ·incRotMatrix+0x0c(SB)/4, $0x00000003 +GLOBL ·incRotMatrix(SB), NOPTR|RODATA, $32 + +#define NUM_ROUNDS 10 + +// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32) +TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0 + MOVV dst+0(FP), R4 + MOVV src+24(FP), R5 + MOVV src_len+32(FP), R6 + MOVV key+48(FP), R7 + MOVV nonce+56(FP), R8 + MOVV counter+64(FP), R9 + + MOVV $·constants(SB), R10 + MOVV $·incRotMatrix(SB), R11 + + MOVW (R9), R12 + +loop: + MOVV $NUM_ROUNDS, R15 + // load 4-32bit data from incRotMatrix added to counter + VMOVQ (R11), V30 + + // load contants + VMOVQ (R10), V0.W4 + VMOVQ 4(R10), V1.W4 + VMOVQ 8(R10), V2.W4 + VMOVQ 12(R10), V3.W4 + + // load keys + VMOVQ (R7), V4.W4 + VMOVQ 4(R7), V5.W4 + VMOVQ 8(R7), V6.W4 + VMOVQ 12(R7), V7.W4 + VMOVQ 16(R7), V8.W4 + VMOVQ 20(R7), V9.W4 + VMOVQ 24(R7), V10.W4 + VMOVQ 28(R7), V11.W4 + + // load counter + nonce + VMOVQ (R9), V12.W4 + + VMOVQ (R8), V13.W4 + VMOVQ 4(R8), V14.W4 + VMOVQ 8(R8), V15.W4 + + // update counter + VADDW V30, V12, V12 + +chacha: + // V0..V3 += V4..V7 + // V12..V15 <<<= ((V12..V15 XOR V0..V3), 16) + VADDW V0, V4, V0 + VADDW V1, V5, V1 + VADDW V2, V6, V2 + VADDW V3, V7, V3 + VXORV V12, V0, V12 + VXORV V13, V1, V13 + VXORV V14, V2, V14 + VXORV V15, V3, V15 + VROTRW $16, V12, V12 + VROTRW $16, V13, V13 + VROTRW $16, V14, V14 + VROTRW $16, V15, V15 + + // V8..V11 += V12..V15 + // V4..V7 <<<= ((V4..V7 XOR V8..V11), 12) + VADDW V8, V12, V8 + VADDW V9, V13, V9 + VADDW V10, V14, V10 + VADDW V11, V15, V11 + VXORV V4, V8, V4 + VXORV V5, V9, V5 + VXORV V6, V10, V6 + VXORV V7, V11, V7 + VROTRW $20, V4, V4 + VROTRW $20, V5, V5 + VROTRW $20, V6, V6 + VROTRW $20, V7, V7 + + // V0..V3 += V4..V7 + // V12..V15 <<<= ((V12..V15 XOR V0..V3), 8) + VADDW V0, V4, V0 + VADDW V1, V5, V1 + VADDW V2, V6, V2 + VADDW V3, V7, V3 + VXORV V12, V0, V12 + VXORV V13, V1, V13 + VXORV V14, V2, V14 + VXORV V15, V3, V15 + VROTRW $24, V12, V12 + VROTRW $24, V13, V13 + VROTRW $24, V14, V14 + VROTRW $24, V15, V15 + + // V8..V11 += V12..V15 + // V4..V7 <<<= ((V4..V7 XOR V8..V11), 7) + VADDW V12, V8, V8 + VADDW V13, V9, V9 + VADDW V14, V10, V10 + VADDW V15, V11, V11 + VXORV V4, V8, V4 + VXORV V5, V9, V5 + VXORV V6, V10, V6 + VXORV V7, V11, V7 + VROTRW $25, V4, V4 + VROTRW $25, V5, V5 + VROTRW $25, V6, V6 + VROTRW $25, V7, V7 + + // V0..V3 += V5..V7, V4 + // V15,V12-V14 <<<= ((V15,V12-V14 XOR V0..V3), 16) + VADDW V0, V5, V0 + VADDW V1, V6, V1 + VADDW V2, V7, V2 + VADDW V3, V4, V3 + VXORV V15, V0, V15 + VXORV V12, V1, V12 + VXORV V13, V2, V13 + VXORV V14, V3, V14 + VROTRW $16, V15, V15 + VROTRW $16, V12, V12 + VROTRW $16, V13, V13 + VROTRW $16, V14, V14 + + // V10,V11,V8,V9 += V15,V12,V13,V14 + // V5,V6,V7,V4 <<<= ((V5,V6,V7,V4 XOR V10,V11,V8,V9), 12) + VADDW V10, V15, V10 + VADDW V11, V12, V11 + VADDW V8, V13, V8 + VADDW V9, V14, V9 + VXORV V5, V10, V5 + VXORV V6, V11, V6 + VXORV V7, V8, V7 + VXORV V4, V9, V4 + VROTRW $20, V5, V5 + VROTRW $20, V6, V6 + VROTRW $20, V7, V7 + VROTRW $20, V4, V4 + + // V0..V3 += V5..V7, V4 + // V15,V12-V14 <<<= ((V15,V12-V14 XOR V0..V3), 8) + VADDW V5, V0, V0 + VADDW V6, V1, V1 + VADDW V7, V2, V2 + VADDW V4, V3, V3 + VXORV V15, V0, V15 + VXORV V12, V1, V12 + VXORV V13, V2, V13 + VXORV V14, V3, V14 + VROTRW $24, V15, V15 + VROTRW $24, V12, V12 + VROTRW $24, V13, V13 + VROTRW $24, V14, V14 + + // V10,V11,V8,V9 += V15,V12,V13,V14 + // V5,V6,V7,V4 <<<= ((V5,V6,V7,V4 XOR V10,V11,V8,V9), 7) + VADDW V15, V10, V10 + VADDW V12, V11, V11 + VADDW V13, V8, V8 + VADDW V14, V9, V9 + VXORV V5, V10, V5 + VXORV V6, V11, V6 + VXORV V7, V8, V7 + VXORV V4, V9, V4 + VROTRW $25, V5, V5 + VROTRW $25, V6, V6 + VROTRW $25, V7, V7 + VROTRW $25, V4, V4 + + SUBV $1, R15 + BNE R15, R0, chacha + + // load origin contants + VMOVQ (R10), V16.W4 + VMOVQ 4(R10), V17.W4 + VMOVQ 8(R10), V18.W4 + VMOVQ 12(R10), V19.W4 + + // load origin keys + VMOVQ (R7), V20.W4 + VMOVQ 4(R7), V21.W4 + VMOVQ 8(R7), V22.W4 + VMOVQ 12(R7), V23.W4 + VMOVQ 16(R7), V24.W4 + VMOVQ 20(R7), V25.W4 + VMOVQ 24(R7), V26.W4 + VMOVQ 28(R7), V27.W4 + + // add back the initial state to generate the key stream + VADDW V30, V12, V12 // update counter in advance to prevent V30 from being overwritten + VADDW V16, V0, V0 + VADDW V17, V1, V1 + VADDW V18, V2, V2 + VADDW V19, V3, V3 + + // load origin counter + nonce + VMOVQ (R9), V28.W4 + VMOVQ (R8), V29.W4 + VMOVQ 4(R8), V30.W4 + VMOVQ 8(R8), V31.W4 + + VADDW V20, V4, V4 + VADDW V21, V5, V5 + VADDW V22, V6, V6 + VADDW V23, V7, V7 + VADDW V24, V8, V8 + VADDW V25, V9, V9 + VADDW V26, V10, V10 + VADDW V27, V11, V11 + VADDW V28, V12, V12 + VADDW V29, V13, V13 + VADDW V30, V14, V14 + VADDW V31, V15, V15 + + // shuffle + VILVLW V0, V1, V16 + VILVHW V0, V1, V17 + VILVLW V2, V3, V18 + VILVHW V2, V3, V19 + VILVLW V4, V5 ,V20 + VILVHW V4, V5, V21 + VILVLW V6, V7, V22 + VILVHW V6, V7, V23 + VILVLW V8, V9, V24 + VILVHW V8, V9, V25 + VILVLW V10, V11, V26 + VILVHW V10, V11, V27 + VILVLW V12, V13, V28 + VILVHW V12, V13, V29 + VILVLW V14, V15, V30 + VILVHW V14, V15, V31 + VILVLV V16, V18, V0 + VILVHV V16, V18, V4 + VILVLV V17, V19, V8 + VILVHV V17, V19, V12 + + // load src data from R5 + VMOVQ 0(R5), V16 + VMOVQ 16(R5), V17 + VMOVQ 32(R5), V18 + VMOVQ 48(R5), V19 + + VILVLV V20, V22, V1 + VILVHV V20, V22, V5 + VILVLV V21, V23, V9 + VILVHV V21, V23, V13 + + VMOVQ 64(R5), V20 + VMOVQ 80(R5), V21 + VMOVQ 96(R5), V22 + VMOVQ 112(R5), V23 + + VILVLV V24, V26, V2 + VILVHV V24, V26, V6 + VILVLV V25, V27, V10 + VILVHV V25, V27, V14 + + VMOVQ 128(R5), V24 + VMOVQ 144(R5), V25 + VMOVQ 160(R5), V26 + VMOVQ 176(R5), V27 + + VILVLV V28, V30, V3 + VILVHV V28, V30, V7 + VILVLV V29, V31, V11 + VILVHV V29, V31, V15 + + VMOVQ 192(R5), V28 + VMOVQ 208(R5), V29 + VMOVQ 224(R5), V30 + VMOVQ 240(R5), V31 + + VXORV V0, V16, V16 + VXORV V1, V17, V17 + VXORV V2, V18, V18 + VXORV V3, V19, V19 + + VMOVQ V16, 0(R4) + VMOVQ V17, 16(R4) + VMOVQ V18, 32(R4) + VMOVQ V19, 48(R4) + + VXORV V4, V20, V20 + VXORV V5, V21, V21 + VXORV V6, V22, V22 + VXORV V7, V23, V23 + + VMOVQ V20, 64(R4) + VMOVQ V21, 80(R4) + VMOVQ V22, 96(R4) + VMOVQ V23, 112(R4) + + VXORV V8, V24, V24 + VXORV V9, V25, V25 + VXORV V10, V26, V26 + VXORV V11, V27, V27 + + VMOVQ V24, 128(R4) + VMOVQ V25, 144(R4) + VMOVQ V26, 160(R4) + VMOVQ V27, 176(R4) + + VXORV V12, V28, V28 + VXORV V13, V29, V29 + VXORV V14, V30, V30 + VXORV V15, V31, V31 + + VMOVQ V28, 192(R4) + VMOVQ V29, 208(R4) + VMOVQ V30, 224(R4) + VMOVQ V31, 240(R4) + + ADD $4, R12, R12 + MOVW R12, (R9) // update counter + + ADDV $256, R4, R4 + ADDV $256, R5, R5 + SUBV $256, R6, R6 + BNE R6, R0, loop + + RET diff --git a/chacha20/chacha_noasm.go b/chacha20/chacha_noasm.go index c709b72..3853cc0 100644 --- a/chacha20/chacha_noasm.go +++ b/chacha20/chacha_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!arm64 && !s390x && !ppc64 && !ppc64le) || !gc || purego +//go:build (!arm64 && !loong64 && !s390x && !ppc64 && !ppc64le) || !gc || purego package chacha20