From a083cb74b68c74fcf1262b6961ac57a6e3b37389 Mon Sep 17 00:00:00 2001 From: XiaolinZhao Date: Wed, 2 Sep 2026 15:34:30 +0800 Subject: [PATCH] argon2: add loong64 LSX implementation of blamka MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmark results on Loongson-3C6000 @ 2.20GHz: goos: linux goarch: loong64 pkg: golang.org/x/crypto/argon2 cpu: Loongson-3C6000/S @ 2200.00MHz | old.bench | new.bench | | sec/op | sec/op vs base | Argon2i/_Time:_3_Memory:_32_MB,_Threads:_1 123.36m ± 0% 76.67m ± 0% -37.85% (p=0.000 n=10) Argon2i/_Time:_4_Memory:_32_MB,_Threads:_1 163.7m ± 0% 101.3m ± 0% -38.09% (p=0.000 n=10) Argon2i/_Time:_5_Memory:_32_MB,_Threads:_1 195.0m ± 0% 116.5m ± 8% -40.24% (p=0.000 n=10) Argon2i/_Time:_3_Memory:_64_MB,_Threads:_4 259.4m ± 0% 165.3m ± 0% -36.28% (p=0.000 n=10) Argon2i/_Time:_4_Memory:_64_MB,_Threads:_4 349.9m ± 0% 220.8m ± 0% -36.92% (p=0.000 n=10) Argon2i/_Time:_5_Memory:_64_MB,_Threads:_4 437.6m ± 0% 276.0m ± 0% -36.92% (p=0.000 n=10) Argon2d/_Time:_3,_Memory:_32_MB,_Threads:_1 116.34m ± 0% 72.07m ± 0% -38.06% (p=0.000 n=10) Argon2d/_Time:_4,_Memory:_32_MB,_Threads:_1 154.42m ± 0% 95.28m ± 0% -38.30% (p=0.000 n=10) Argon2d/_Time:_5,_Memory:_32_MB,_Threads:_1 192.4m ± 0% 115.9m ± 2% -39.73% (p=0.000 n=10) Argon2d/_Time:_3,_Memory:_64_MB,_Threads:_4 254.4m ± 0% 165.2m ± 0% -35.06% (p=0.000 n=10) Argon2d/_Time:_4,_Memory:_64_MB,_Threads:_4 345.5m ± 1% 221.0m ± 0% -36.04% (p=0.000 n=10) Argon2d/_Time:_5,_Memory:_64_MB,_Threads:_4 432.1m ± 0% 276.2m ± 0% -36.09% (p=0.000 n=10) Argon2id/_Time:_3,_Memory:_32_MB,_Threads:_1 116.49m ± 0% 72.38m ± 0% -37.86% (p=0.000 n=10) Argon2id/_Time:_4,_Memory:_32_MB,_Threads:_1 154.59m ± 0% 95.71m ± 3% -38.09% (p=0.000 n=10) Argon2id/_Time:_5,_Memory:_32_MB,_Threads:_1 192.5m ± 0% 115.0m ± 0% -40.25% (p=0.000 n=10) Argon2id/_Time:_3,_Memory:_64_MB,_Threads:_4 255.6m ± 0% 165.7m ± 0% -35.18% (p=0.000 n=10) Argon2id/_Time:_4,_Memory:_64_MB,_Threads:_4 345.4m ± 0% 221.5m ± 0% -35.87% (p=0.000 n=10) Argon2id/_Time:_5,_Memory:_64_MB,_Threads:_4 431.9m ± 0% 276.6m ± 0% -35.96% (p=0.000 n=10) geomean 227.7m 142.6m -37.40% | old.bench | new.bench | | B/op | B/op vs base | Argon2i/_Time:_3_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_4_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_5_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_3_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_4_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_5_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_3,_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_4,_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_5,_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_3,_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_4,_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_5,_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_3,_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_4,_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_5,_Memory:_32_MB,_Threads:_1 32.00Mi ± 0% 32.00Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_3,_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_4,_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_5,_Memory:_64_MB,_Threads:_4 64.01Mi ± 0% 64.01Mi ± 0% ~ (p=1.000 n=10) ¹ geomean 45.26Mi 45.26Mi +0.00% ¹ all samples are equal | old.bench | new.bench | | allocs/op | allocs/op vs base | Argon2i/_Time:_3_Memory:_32_MB,_Threads:_1 31.00 ± 0% 31.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_4_Memory:_32_MB,_Threads:_1 39.00 ± 0% 39.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_5_Memory:_32_MB,_Threads:_1 47.00 ± 0% 47.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_3_Memory:_64_MB,_Threads:_4 73.00 ± 0% 73.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_4_Memory:_64_MB,_Threads:_4 93.00 ± 0% 93.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2i/_Time:_5_Memory:_64_MB,_Threads:_4 113.0 ± 0% 113.0 ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_3,_Memory:_32_MB,_Threads:_1 31.00 ± 0% 31.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_4,_Memory:_32_MB,_Threads:_1 39.00 ± 0% 39.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_5,_Memory:_32_MB,_Threads:_1 47.00 ± 0% 47.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_3,_Memory:_64_MB,_Threads:_4 73.00 ± 0% 73.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_4,_Memory:_64_MB,_Threads:_4 93.00 ± 0% 93.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2d/_Time:_5,_Memory:_64_MB,_Threads:_4 113.0 ± 0% 113.0 ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_3,_Memory:_32_MB,_Threads:_1 31.00 ± 0% 31.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_4,_Memory:_32_MB,_Threads:_1 39.00 ± 0% 39.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_5,_Memory:_32_MB,_Threads:_1 47.00 ± 0% 47.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_3,_Memory:_64_MB,_Threads:_4 73.00 ± 0% 73.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_4,_Memory:_64_MB,_Threads:_4 93.00 ± 0% 93.00 ± 0% ~ (p=1.000 n=10) ¹ Argon2id/_Time:_5,_Memory:_64_MB,_Threads:_4 113.0 ± 0% 113.0 ± 0% ~ (p=1.000 n=10) ¹ geomean 59.32 59.32 +0.00% ¹ all samples are equal Change-Id: Ia814057ddfdefbe1d23431fa366189ffe5852b96 Reviewed-on: https://go-review.googlesource.com/c/crypto/+/826304 Reviewed-by: Junyang Shao LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: abner chenc Reviewed-by: Cherry Mui --- argon2/argon2_test.go | 9 ++ argon2/blamka_generic.go | 4 + argon2/blamka_loong64.go | 49 ++++++++ argon2/blamka_loong64.s | 240 +++++++++++++++++++++++++++++++++++++++ argon2/blamka_ref.go | 2 +- 5 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 argon2/blamka_loong64.go create mode 100644 argon2/blamka_loong64.s diff --git a/argon2/argon2_test.go b/argon2/argon2_test.go index 775b97a..662c3bc 100644 --- a/argon2/argon2_test.go +++ b/argon2/argon2_test.go @@ -24,6 +24,7 @@ var ( func TestArgon2(t *testing.T) { defer func(sse4 bool) { useSSE4 = sse4 }(useSSE4) + defer func(lsx bool) { useLSX = lsx }(useLSX) if useSSE4 { t.Log("SSE4.1 version") @@ -32,6 +33,14 @@ func TestArgon2(t *testing.T) { testArgon2id(t) useSSE4 = false } + + if useLSX { + t.Log("LSX version") + testArgon2i(t) + testArgon2d(t) + testArgon2id(t) + useLSX = false + } t.Log("generic version") testArgon2i(t) testArgon2d(t) diff --git a/argon2/blamka_generic.go b/argon2/blamka_generic.go index a481b22..4c9dabb 100644 --- a/argon2/blamka_generic.go +++ b/argon2/blamka_generic.go @@ -6,6 +6,10 @@ package argon2 var useSSE4 bool +// useLSX reports whether the loong64 LSX vector implementation should be +// used. It is set from cpu.Loong64.HasLSX in blamka_loong64.go's init(). +var useLSX bool + func processBlockGeneric(out, in1, in2 *block, xor bool) { var t block for i := range t { diff --git a/argon2/blamka_loong64.go b/argon2/blamka_loong64.go new file mode 100644 index 0000000..e7daeb4 --- /dev/null +++ b/argon2/blamka_loong64.go @@ -0,0 +1,49 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && gc && !purego + +package argon2 + +import "golang.org/x/sys/cpu" + +func init() { + // useLSX (declared in blamka_generic.go) gates the LSX vector path. + // If the running CPU lacks LSX support, processBlockVX falls back + // to processBlockGeneric. + useLSX = cpu.Loong64.HasLSX +} + +//go:noescape +func blamkaLOONG64(b *block) + +//go:noescape +func mixBlocksVX(out, a, b, c *block) + +//go:noescape +func xorBlocksVX(out, a, b, c *block) + +func processBlock(out, in1, in2 *block) { + processBlockVX(out, in1, in2, false) +} + +func processBlockXOR(out, in1, in2 *block) { + processBlockVX(out, in1, in2, true) +} + +func processBlockVX(out, in1, in2 *block, xor bool) { + if !useLSX { + processBlockGeneric(out, in1, in2, xor) + return + } + + var t block + mixBlocksVX(&t, in1, in2, &t) + blamkaLOONG64(&t) + if xor { + xorBlocksVX(out, in1, in2, &t) + } else { + mixBlocksVX(out, in1, in2, &t) + } +} diff --git a/argon2/blamka_loong64.s b/argon2/blamka_loong64.s new file mode 100644 index 0000000..7843a7c --- /dev/null +++ b/argon2/blamka_loong64.s @@ -0,0 +1,240 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && gc && !purego + +#include "textflag.h" + +// HALF_ROUND applies one BlaMka half-round (mul-add-xor-rotate) to two +// groups of 4 columns, each packed 2-wide into V registers (v0/v2/v4/v6 +// hold one pair of columns in lane0/lane1, v1/v3/v5/v7 hold the other). +// t0, t1 are scratch registers, one per independent column group. +#define HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, t1) \ + VMULWEVVWU v0, v2, t0; \ + VADDV t0, t0, t0; \ + VADDV v0, v2, v0; \ + VADDV v0, t0, v0; \ + VXORV v6, v0, v6; \ + VROTRV $32, v6, v6; \ + VMULWEVVWU v1, v3, t1; \ + VADDV t1, t1, t1; \ + VADDV v1, v3, v1; \ + VADDV v1, t1, v1; \ + VXORV v7, v1, v7; \ + VROTRV $32, v7, v7; \ + VMULWEVVWU v4, v6, t0; \ + VADDV t0, t0, t0; \ + VADDV v4, v6, v4; \ + VADDV v4, t0, v4; \ + VXORV v2, v4, v2; \ + VROTRV $24, v2, v2; \ + VMULWEVVWU v5, v7, t1; \ + VADDV t1, t1, t1; \ + VADDV v5, v7, v5; \ + VADDV v5, t1, v5; \ + VXORV v3, v5, v3; \ + VROTRV $24, v3, v3; \ + VMULWEVVWU v0, v2, t0; \ + VADDV t0, t0, t0; \ + VADDV v0, v2, v0; \ + VADDV v0, t0, v0; \ + VXORV v6, v0, v6; \ + VROTRV $16, v6, v6; \ + VMULWEVVWU v1, v3, t1; \ + VADDV t1, t1, t1; \ + VADDV v1, v3, v1; \ + VADDV v1, t1, v1; \ + VXORV v7, v1, v7; \ + VROTRV $16, v7, v7; \ + VMULWEVVWU v4, v6, t0; \ + VADDV t0, t0, t0; \ + VADDV v4, v6, v4; \ + VADDV v4, t0, v4; \ + VXORV v2, v4, v2; \ + VROTRV $63, v2, v2; \ + VMULWEVVWU v5, v7, t1; \ + VADDV t1, t1, t1; \ + VADDV v5, v7, v5; \ + VADDV v5, t1, v5; \ + VXORV v3, v5, v3; \ + VROTRV $63, v3, v3 + +// SHUFFLE / SHUFFLE_INV: reused verbatim from blake2b_loong64.s -- amd64 +// avo source confirms argon2's SHUFFLE/SHUFFLE_INV and blake2b's SHUFFLE +// are byte-for-byte identical macros (diagonalize / undiagonalize), +// operating only on V2..V7 (V0/V1 untouched), with V13 held at zero. +#define SHUFFLE \ + VADDV V2, V13, V12; \ + VADDV V6, V13, V16; \ + VADDV V4, V13, V14; \ + VADDV V5, V13, V4; \ + VADDV V14, V13, V5; \ + VSHUF4IV $9, V3, V2; \ + VSHUF4IV $9, V12, V3; \ + VSHUF4IV $3, V7, V6; \ + VSHUF4IV $3, V16, V7 + +#define SHUFFLE_INV \ + VADDV V2, V13, V12; \ + VADDV V6, V13, V16; \ + VADDV V4, V13, V14; \ + VADDV V5, V13, V4; \ + VADDV V14, V13, V5; \ + VSHUF4IV $3, V3, V2; \ + VSHUF4IV $3, V12, V3; \ + VSHUF4IV $9, V7, V6; \ + VSHUF4IV $9, V16, V7 + +// LOAD_MSG_0 / STORE_MSG_0: row pass. `off` is the byte offset of the +// 128-byte (16-uint64) row group; within the macro the 8 loads step by +// 16B (2 uint64 per V register). Called from blamkaLOONG64 with off +// stepping by 128B across the 8 row groups -- do not confuse the two +// strides. +#define LOAD_MSG_0(off) \ + VMOVQ (off+0)(R4), V0; \ + VMOVQ (off+16)(R4), V1; \ + VMOVQ (off+32)(R4), V2; \ + VMOVQ (off+48)(R4), V3; \ + VMOVQ (off+64)(R4), V4; \ + VMOVQ (off+80)(R4), V5; \ + VMOVQ (off+96)(R4), V6; \ + VMOVQ (off+112)(R4), V7 + +#define STORE_MSG_0(off) \ + VMOVQ V0, (off+0)(R4); \ + VMOVQ V1, (off+16)(R4); \ + VMOVQ V2, (off+32)(R4); \ + VMOVQ V3, (off+48)(R4); \ + VMOVQ V4, (off+64)(R4); \ + VMOVQ V5, (off+80)(R4); \ + VMOVQ V6, (off+96)(R4); \ + VMOVQ V7, (off+112)(R4) + +// LOAD_MSG_1 / STORE_MSG_1: diagonal pass. `off` is the byte offset within +// a 128-byte row group; within the macro the 8 loads step by 128B (one +// element from each of the 8 row groups, mirroring blamka_generic.go's +// t[off], t[128+off], t[256+off], ...). Called from blamkaLOONG64 with +// off stepping by 16B (2 uint64) across the 8 diagonal groups -- the +// inner/outer strides are swapped relative to LOAD_MSG_0/STORE_MSG_0. +#define LOAD_MSG_1(off) \ + VMOVQ (off+0)(R4), V0; \ + VMOVQ (off+128)(R4), V1; \ + VMOVQ (off+256)(R4), V2; \ + VMOVQ (off+384)(R4), V3; \ + VMOVQ (off+512)(R4), V4; \ + VMOVQ (off+640)(R4), V5; \ + VMOVQ (off+768)(R4), V6; \ + VMOVQ (off+896)(R4), V7 + +#define STORE_MSG_1(off) \ + VMOVQ V0, (off+0)(R4); \ + VMOVQ V1, (off+128)(R4); \ + VMOVQ V2, (off+256)(R4); \ + VMOVQ V3, (off+384)(R4); \ + VMOVQ V4, (off+512)(R4); \ + VMOVQ V5, (off+640)(R4); \ + VMOVQ V6, (off+768)(R4); \ + VMOVQ V7, (off+896)(R4) + +// BLAMKA_ROUND_0(off): one full row pass over the 16-uint64 group at +// byte offset off -- corresponds to a single call to blamkaGeneric +// with the "row" argument order (t[off]..t[off+15]). +#define BLAMKA_ROUND_0(off) \ + LOAD_MSG_0(off); \ + HALF_ROUND(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9); \ + SHUFFLE; \ + HALF_ROUND(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9); \ + SHUFFLE_INV; \ + STORE_MSG_0(off) + +// BLAMKA_ROUND_1(off): one full diagonal pass over the 16 uint64s at +// diagonal offset off -- corresponds to a single call to blamkaGeneric +// with the "diagonal" argument order (t[off], t[off+1], t[16+off], ...). +#define BLAMKA_ROUND_1(off) \ + LOAD_MSG_1(off); \ + HALF_ROUND(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9); \ + SHUFFLE; \ + HALF_ROUND(V0, V1, V2, V3, V4, V5, V6, V7, V8, V9); \ + SHUFFLE_INV; \ + STORE_MSG_1(off) + +// func blamkaLOONG64(b *block) +TEXT ·blamkaLOONG64(SB), NOSPLIT, $0-8 + MOVV b+0(FP), R4 + + VXORV V13, V13, V13 // V13 = 0, used by SHUFFLE/SHUFFLE_INV + + // Round 0: 8 row passes, stride 128 bytes (16 uint64s). + BLAMKA_ROUND_0(0) + BLAMKA_ROUND_0(128) + BLAMKA_ROUND_0(256) + BLAMKA_ROUND_0(384) + BLAMKA_ROUND_0(512) + BLAMKA_ROUND_0(640) + BLAMKA_ROUND_0(768) + BLAMKA_ROUND_0(896) + + // Round 1: 8 diagonal passes, stride 16 bytes (2 uint64s). + BLAMKA_ROUND_1(0) + BLAMKA_ROUND_1(16) + BLAMKA_ROUND_1(32) + BLAMKA_ROUND_1(48) + BLAMKA_ROUND_1(64) + BLAMKA_ROUND_1(80) + BLAMKA_ROUND_1(96) + BLAMKA_ROUND_1(112) + + RET + +// func mixBlocksVX(out, a, b, c *block) +TEXT ·mixBlocksVX(SB), NOSPLIT, $0-32 + MOVV out+0(FP), R4 + MOVV a+8(FP), R5 + MOVV b+16(FP), R6 + MOVV c+24(FP), R7 + MOVV $1024, R8 + + PCALIGN $16 +mixloop: + VMOVQ (R5), V0 + VMOVQ (R6), V1 + VMOVQ (R7), V2 + VXORV V0, V1, V0 + VXORV V0, V2, V0 + VMOVQ V0, (R4) + ADDV $16, R4 + ADDV $16, R5 + ADDV $16, R6 + ADDV $16, R7 + SUBV $16, R8 + BNE R8, R0, mixloop + + RET + +// func xorBlocksVX(out, a, b, c *block) +TEXT ·xorBlocksVX(SB), NOSPLIT, $0-32 + MOVV out+0(FP), R4 + MOVV a+8(FP), R5 + MOVV b+16(FP), R6 + MOVV c+24(FP), R7 + MOVV $1024, R8 + + PCALIGN $16 +xorloop: + VMOVQ (R5), V0 + VMOVQ (R6), V1 + VMOVQ (R7), V2 + VMOVQ (R4), V3 + VXORV V0, V1, V0 + VXORV V0, V2, V0 + VXORV V0, V3, V0 + VMOVQ V0, (R4) + ADDV $16, R4 + ADDV $16, R5 + ADDV $16, R6 + ADDV $16, R7 + SUBV $16, R8 + BNE R8, R0, xorloop + + RET diff --git a/argon2/blamka_ref.go b/argon2/blamka_ref.go index 16d58c6..cf3e141 100644 --- a/argon2/blamka_ref.go +++ b/argon2/blamka_ref.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 || purego || !gc +//go:build (!amd64 && !loong64) || purego || !gc package argon2