go.mod: update golang.org/x dependencies

Update golang.org/x dependencies to their latest tagged versions.

Change-Id: Id321d3b5909ecb66c0311ba86008509c7895863b
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/617958
Auto-Submit: Gopher Robot <gobot@golang.org>
Reviewed-by: David Chase <drchase@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>

acme/autocert/internal/acmetest/ca.go

@@ -308,7 +308,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		}
 
 		if err := decodePayload(&req, r.Body); err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 
@@ -328,7 +328,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 			Identifiers []struct{ Value string }
 		}
 		if err := decodePayload(&req, r.Body); err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		ca.mu.Lock()
@@ -352,7 +352,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		defer ca.mu.Unlock()
 		o, err := ca.storedOrder(strings.TrimPrefix(r.URL.Path, "/orders/"))
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		if err := json.NewEncoder(w).Encode(o); err != nil {
@@ -412,7 +412,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		orderID := strings.TrimPrefix(r.URL.Path, "/new-cert/")
 		o, err := ca.storedOrder(orderID)
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		if o.Status != acme.StatusReady {
@@ -427,7 +427,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		b, _ := base64.RawURLEncoding.DecodeString(req.CSR)
 		csr, err := x509.ParseCertificateRequest(b)
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		// Issue the certificate.
@@ -449,7 +449,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		defer ca.mu.Unlock()
 		o, err := ca.storedOrder(strings.TrimPrefix(r.URL.Path, "/issued-cert/"))
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		if o.Status != acme.StatusValid {

argon2/_asm/blamka_amd64.go

@@ -0,0 +1,287 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/argon2"
+)
+
+//go:generate go run . -out ../blamka_amd64.s -pkg argon2
+
+func main() {
+	Package("golang.org/x/crypto/argon2")
+	ConstraintExpr("amd64,gc,!purego")
+
+	blamkaSSE4()
+	mixBlocksSSE2()
+	xorBlocksSSE2()
+	Generate()
+}
+
+func blamkaSSE4() {
+	Implement("blamkaSSE4")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("b"), RAX)
+
+	c40 := c40_DATA()
+	c48 := c48_DATA()
+
+	MOVOU(c40, X10)
+	MOVOU(c48, X11)
+
+	BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11)
+
+	BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11)
+	RET()
+}
+
+func mixBlocksSSE2() {
+	Implement("mixBlocksSSE2")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("out"), RDX)
+	Load(Param("a"), RAX)
+	Load(Param("b"), RBX)
+	Load(Param("c"), RCX)
+	MOVQ(U32(128), RDI)
+
+	Label("loop")
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: BX}.Offset(0), X1)
+	MOVOU(Mem{Base: CX}.Offset(0), X2)
+	PXOR(X1, X0)
+	PXOR(X2, X0)
+	MOVOU(X0, Mem{Base: DX}.Offset(0))
+	ADDQ(Imm(16), RAX)
+	ADDQ(Imm(16), RBX)
+	ADDQ(Imm(16), RCX)
+	ADDQ(Imm(16), RDX)
+	SUBQ(Imm(2), RDI)
+	JA(LabelRef("loop"))
+	RET()
+}
+
+func xorBlocksSSE2() {
+	Implement("xorBlocksSSE2")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("out"), RDX)
+	Load(Param("a"), RAX)
+	Load(Param("b"), RBX)
+	Load(Param("c"), RCX)
+	MOVQ(U32(128), RDI)
+
+	Label("loop")
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: BX}.Offset(0), X1)
+	MOVOU(Mem{Base: CX}.Offset(0), X2)
+	MOVOU(Mem{Base: DX}.Offset(0), X3)
+	PXOR(X1, X0)
+	PXOR(X2, X0)
+	PXOR(X3, X0)
+	MOVOU(X0, Mem{Base: DX}.Offset(0))
+	ADDQ(Imm(16), RAX)
+	ADDQ(Imm(16), RBX)
+	ADDQ(Imm(16), RCX)
+	ADDQ(Imm(16), RDX)
+	SUBQ(Imm(2), RDI)
+	JA(LabelRef("loop"))
+	RET()
+}
+
+func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v6, t1)
+	PUNPCKLQDQ(v6, t2)
+	PUNPCKHQDQ(v7, v6)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(v7, t2)
+	MOVO(t1, v7)
+	MOVO(v2, t1)
+	PUNPCKHQDQ(t2, v7)
+	PUNPCKLQDQ(v3, t2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v3)
+}
+
+func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v2, t1)
+	PUNPCKLQDQ(v2, t2)
+	PUNPCKHQDQ(v3, v2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(v3, t2)
+	MOVO(t1, v3)
+	MOVO(v6, t1)
+	PUNPCKHQDQ(t2, v3)
+	PUNPCKLQDQ(v7, t2)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v7)
+}
+
+func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48 VecPhysical) {
+	MOVO(v0, t0)
+	PMULULQ(v2, t0)
+	PADDQ(v2, v0)
+	PADDQ(t0, v0)
+	PADDQ(t0, v0)
+	PXOR(v0, v6)
+	PSHUFD(Imm(0xB1), v6, v6)
+	MOVO(v4, t0)
+	PMULULQ(v6, t0)
+	PADDQ(v6, v4)
+	PADDQ(t0, v4)
+	PADDQ(t0, v4)
+	PXOR(v4, v2)
+	PSHUFB(c40, v2)
+	MOVO(v0, t0)
+	PMULULQ(v2, t0)
+	PADDQ(v2, v0)
+	PADDQ(t0, v0)
+	PADDQ(t0, v0)
+	PXOR(v0, v6)
+	PSHUFB(c48, v6)
+	MOVO(v4, t0)
+	PMULULQ(v6, t0)
+	PADDQ(v6, v4)
+	PADDQ(t0, v4)
+	PADDQ(t0, v4)
+	PXOR(v4, v2)
+	MOVO(v2, t0)
+	PADDQ(v2, t0)
+	PSRLQ(Imm(63), v2)
+	PXOR(t0, v2)
+	MOVO(v1, t0)
+	PMULULQ(v3, t0)
+	PADDQ(v3, v1)
+	PADDQ(t0, v1)
+	PADDQ(t0, v1)
+	PXOR(v1, v7)
+	PSHUFD(Imm(0xB1), v7, v7)
+	MOVO(v5, t0)
+	PMULULQ(v7, t0)
+	PADDQ(v7, v5)
+	PADDQ(t0, v5)
+	PADDQ(t0, v5)
+	PXOR(v5, v3)
+	PSHUFB(c40, v3)
+	MOVO(v1, t0)
+	PMULULQ(v3, t0)
+	PADDQ(v3, v1)
+	PADDQ(t0, v1)
+	PADDQ(t0, v1)
+	PXOR(v1, v7)
+	PSHUFB(c48, v7)
+	MOVO(v5, t0)
+	PMULULQ(v7, t0)
+	PADDQ(v7, v5)
+	PADDQ(t0, v5)
+	PADDQ(t0, v5)
+	PXOR(v5, v3)
+	MOVO(v3, t0)
+	PADDQ(v3, t0)
+	PSRLQ(Imm(63), v3)
+	PXOR(t0, v3)
+}
+
+func LOAD_MSG_0(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(Mem{Base: block}.Offset(8*(off+(i*2))), r)
+	}
+}
+
+func STORE_MSG_0(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(r, Mem{Base: block}.Offset(8*(off+(i*2))))
+	}
+}
+
+func LOAD_MSG_1(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(Mem{Base: block}.Offset(8*off+i*16*8), r)
+	}
+}
+
+func STORE_MSG_1(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(r, Mem{Base: block}.Offset(8*off+i*16*8))
+	}
+}
+
+func BLAMKA_ROUND_0(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) {
+	LOAD_MSG_0(block, off)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1)
+	STORE_MSG_0(block, off)
+}
+
+func BLAMKA_ROUND_1(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) {
+	LOAD_MSG_1(block, off)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1)
+	STORE_MSG_1(block, off)
+}
+
+// ##------------------DATA SECTION-------------------##
+
+var c40_DATA_ptr, c48_DATA_ptr *Mem
+
+func c40_DATA() Mem {
+	if c40_DATA_ptr != nil {
+		return *c40_DATA_ptr
+	}
+
+	c40_DATA := GLOBL("·c40", NOPTR|RODATA)
+	c40_DATA_ptr = &c40_DATA
+	DATA(0x00, U64(0x0201000706050403))
+	DATA(0x08, U64(0x0a09080f0e0d0c0b))
+	return c40_DATA
+}
+func c48_DATA() Mem {
+	if c48_DATA_ptr != nil {
+		return *c48_DATA_ptr
+	}
+
+	c48_DATA := GLOBL("·c48", NOPTR|RODATA)
+	c48_DATA_ptr = &c48_DATA
+	DATA(0x00, U64(0x0100070605040302))
+	DATA(0x08, U64(0x09080f0e0d0c0b0a))
+	return c48_DATA
+}

argon2/_asm/go.mod

@@ -0,0 +1,15 @@
+module argon2/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

argon2/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

blake2b/_asm/AVX2/go.mod

@@ -0,0 +1,16 @@
+module blake2b/_asm/AVX2
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

blake2b/_asm/AVX2/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

blake2b/_asm/standard/blake2b_amd64_asm.go

@@ -0,0 +1,361 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/blake2b"
+)
+
+//go:generate go run . -out ../../blake2b_amd64.s -pkg blake2b
+
+const ThatPeskyUnicodeDot = "\u00b7"
+
+var iv0_DATA_ptr, iv1_DATA_ptr, iv2_DATA_ptr, iv3_DATA_ptr, c40_DATA_ptr, c48_DATA_ptr *Mem
+
+func main() {
+	Package("golang.org/x/crypto/blake2b")
+	ConstraintExpr("amd64,gc,!purego")
+	hashBlocksSSE4()
+	Generate()
+}
+
+func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v6, t1)
+	PUNPCKLQDQ(v6, t2)
+	PUNPCKHQDQ(v7, v6)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(v7, t2)
+	MOVO(t1, v7)
+	MOVO(v2, t1)
+	PUNPCKHQDQ(t2, v7)
+	PUNPCKLQDQ(v3, t2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v3)
+}
+
+func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v2, t1)
+	PUNPCKLQDQ(v2, t2)
+	PUNPCKHQDQ(v3, v2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(v3, t2)
+	MOVO(t1, v3)
+	MOVO(v6, t1)
+	PUNPCKHQDQ(t2, v3)
+	PUNPCKLQDQ(v7, t2)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v7)
+}
+
+func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7 VecPhysical, m0, m1, m2, m3 Op, t0, c40, c48 VecPhysical) {
+	PADDQ(m0, v0)
+	PADDQ(m1, v1)
+	PADDQ(v2, v0)
+	PADDQ(v3, v1)
+	PXOR(v0, v6)
+	PXOR(v1, v7)
+	PSHUFD(Imm(0xB1), v6, v6)
+	PSHUFD(Imm(0xB1), v7, v7)
+	PADDQ(v6, v4)
+	PADDQ(v7, v5)
+	PXOR(v4, v2)
+	PXOR(v5, v3)
+	PSHUFB(c40, v2)
+	PSHUFB(c40, v3)
+	PADDQ(m2, v0)
+	PADDQ(m3, v1)
+	PADDQ(v2, v0)
+	PADDQ(v3, v1)
+	PXOR(v0, v6)
+	PXOR(v1, v7)
+	PSHUFB(c48, v6)
+	PSHUFB(c48, v7)
+	PADDQ(v6, v4)
+	PADDQ(v7, v5)
+	PXOR(v4, v2)
+	PXOR(v5, v3)
+	MOVOU(v2, t0)
+	PADDQ(v2, t0)
+	PSRLQ(Imm(63), v2)
+	PXOR(t0, v2)
+	MOVOU(v3, t0)
+	PADDQ(v3, t0)
+	PSRLQ(Imm(63), v3)
+	PXOR(t0, v3)
+}
+
+func LOAD_MSG(m0, m1, m2, m3 VecPhysical, src GPPhysical, i0, i1, i2, i3, i4, i5, i6, i7 int) {
+	MOVQ(Mem{Base: src}.Offset(i0*8), m0)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i1*8), m0)
+	MOVQ(Mem{Base: src}.Offset(i2*8), m1)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i3*8), m1)
+	MOVQ(Mem{Base: src}.Offset(i4*8), m2)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i5*8), m2)
+	MOVQ(Mem{Base: src}.Offset(i6*8), m3)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i7*8), m3)
+}
+
+func hashBlocksSSE4() {
+	Implement("hashBlocksSSE4")
+	Attributes(4)
+	AllocLocal(288) // frame size = 272 + 16 byte alignment
+
+	Load(Param("h"), RAX)
+	Load(Param("c"), RBX)
+	Load(Param("flag"), RCX)
+	Load(Param("blocks").Base(), RSI)
+	Load(Param("blocks").Len(), RDI)
+
+	MOVQ(RSP, R10)
+	ADDQ(Imm(15), R10)
+	ANDQ(I32(-16), R10)
+
+	iv3 := iv3_DATA()
+	MOVOU(iv3, X0)
+	MOVO(X0, Mem{Base: R10}.Offset(0))
+	XORQ(RCX, Mem{Base: R10}.Offset(0)) // 0(R10) = ·iv3 ^ (CX || 0)
+
+	c40 := c40_DATA()
+	c48 := c48_DATA()
+	MOVOU(c40, X13)
+	MOVOU(c48, X14)
+
+	MOVOU(Mem{Base: AX}.Offset(0), X12)
+	MOVOU(Mem{Base: AX}.Offset(16), X15)
+
+	MOVQ(Mem{Base: BX}.Offset(0), R8)
+	MOVQ(Mem{Base: BX}.Offset(8), R9)
+
+	Label("loop")
+	ADDQ(Imm(128), R8)
+	CMPQ(R8, Imm(128))
+	JGE(LabelRef("noinc"))
+	INCQ(R9)
+
+	Label("noinc")
+	MOVQ(R8, X8)
+	PINSRQ(Imm(1), R9, X8)
+
+	iv0 := iv0_DATA()
+	iv1 := iv1_DATA()
+	iv2 := iv2_DATA()
+
+	MOVO(X12, X0)
+	MOVO(X15, X1)
+	MOVOU(Mem{Base: AX}.Offset(32), X2)
+	MOVOU(Mem{Base: AX}.Offset(48), X3)
+	MOVOU(iv0, X4)
+	MOVOU(iv1, X5)
+	MOVOU(iv2, X6)
+
+	PXOR(X8, X6)
+	MOVO(Mem{Base: R10}.Offset(0), X7)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7)
+	MOVO(X8, Mem{Base: R10}.Offset(16))
+	MOVO(X9, Mem{Base: R10}.Offset(32))
+	MOVO(X10, Mem{Base: R10}.Offset(48))
+	MOVO(X11, Mem{Base: R10}.Offset(64))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 8, 10, 12, 14, 9, 11, 13, 15)
+	MOVO(X8, Mem{Base: R10}.Offset(80))
+	MOVO(X9, Mem{Base: R10}.Offset(96))
+	MOVO(X10, Mem{Base: R10}.Offset(112))
+	MOVO(X11, Mem{Base: R10}.Offset(128))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6)
+	MOVO(X8, Mem{Base: R10}.Offset(144))
+	MOVO(X9, Mem{Base: R10}.Offset(160))
+	MOVO(X10, Mem{Base: R10}.Offset(176))
+	MOVO(X11, Mem{Base: R10}.Offset(192))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 1, 0, 11, 5, 12, 2, 7, 3)
+	MOVO(X8, Mem{Base: R10}.Offset(208))
+	MOVO(X9, Mem{Base: R10}.Offset(224))
+	MOVO(X10, Mem{Base: R10}.Offset(240))
+	MOVO(X11, Mem{Base: R10}.Offset(256))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 10, 3, 7, 9, 14, 6, 1, 4)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 2, 5, 4, 15, 6, 10, 0, 8)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 14, 11, 6, 3, 1, 12, 8, 13)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 4, 7, 15, 1, 13, 5, 14, 9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 0, 6, 9, 8, 7, 3, 2, 11)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 5, 15, 8, 2, 0, 4, 6, 10)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 12, 13, 1, 10, 2, 7, 4, 5)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 15, 9, 3, 13, 11, 14, 12, 0)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(16), Mem{Base: R10}.Offset(32), Mem{Base: R10}.Offset(48), Mem{Base: R10}.Offset(64), X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(80), Mem{Base: R10}.Offset(96), Mem{Base: R10}.Offset(112), Mem{Base: R10}.Offset(128), X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(144), Mem{Base: R10}.Offset(160), Mem{Base: R10}.Offset(176), Mem{Base: R10}.Offset(192), X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(208), Mem{Base: R10}.Offset(224), Mem{Base: R10}.Offset(240), Mem{Base: R10}.Offset(256), X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	MOVOU(Mem{Base: AX}.Offset(32), X10)
+	MOVOU(Mem{Base: AX}.Offset(48), X11)
+	PXOR(X0, X12)
+	PXOR(X1, X15)
+	PXOR(X2, X10)
+	PXOR(X3, X11)
+	PXOR(X4, X12)
+	PXOR(X5, X15)
+	PXOR(X6, X10)
+	PXOR(X7, X11)
+	MOVOU(X10, Mem{Base: AX}.Offset(32))
+	MOVOU(X11, Mem{Base: AX}.Offset(48))
+
+	LEAQ(Mem{Base: SI}.Offset(128), RSI)
+	SUBQ(Imm(128), RDI)
+	JNE(LabelRef("loop"))
+
+	MOVOU(X12, Mem{Base: AX}.Offset(0))
+	MOVOU(X15, Mem{Base: AX}.Offset(16))
+
+	MOVQ(R8, Mem{Base: BX}.Offset(0))
+	MOVQ(R9, Mem{Base: BX}.Offset(8))
+
+	RET()
+}
+
+// #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
+
+func iv0_DATA() Mem {
+	if iv0_DATA_ptr != nil {
+		return *iv0_DATA_ptr
+	}
+
+	iv0 := GLOBL(ThatPeskyUnicodeDot+"iv0", NOPTR|RODATA)
+	iv0_DATA_ptr = &iv0
+	DATA(0x00, U64(0x6a09e667f3bcc908))
+	DATA(0x08, U64(0xbb67ae8584caa73b))
+	return iv0
+}
+
+func iv1_DATA() Mem {
+	if iv1_DATA_ptr != nil {
+		return *iv1_DATA_ptr
+	}
+
+	iv1 := GLOBL(ThatPeskyUnicodeDot+"iv1", NOPTR|RODATA)
+	iv1_DATA_ptr = &iv1
+	DATA(0x00, U64(0x3c6ef372fe94f82b))
+	DATA(0x08, U64(0xa54ff53a5f1d36f1))
+	return iv1
+}
+
+func iv2_DATA() Mem {
+	if iv2_DATA_ptr != nil {
+		return *iv2_DATA_ptr
+	}
+
+	iv2 := GLOBL(ThatPeskyUnicodeDot+"iv2", NOPTR|RODATA)
+	iv2_DATA_ptr = &iv2
+	DATA(0x00, U64(0x510e527fade682d1))
+	DATA(0x08, U64(0x9b05688c2b3e6c1f))
+	return iv2
+}
+
+func iv3_DATA() Mem {
+	if iv3_DATA_ptr != nil {
+		return *iv3_DATA_ptr
+	}
+
+	iv3 := GLOBL(ThatPeskyUnicodeDot+"iv3", NOPTR|RODATA)
+	iv3_DATA_ptr = &iv3
+	DATA(0x00, U64(0x1f83d9abfb41bd6b))
+	DATA(0x08, U64(0x5be0cd19137e2179))
+	return iv3
+}
+
+func c40_DATA() Mem {
+	if c40_DATA_ptr != nil {
+		return *c40_DATA_ptr
+	}
+
+	c40 := GLOBL(ThatPeskyUnicodeDot+"c40", NOPTR|RODATA)
+	c40_DATA_ptr = &c40
+	DATA(0x00, U64(0x0201000706050403))
+	DATA(0x08, U64(0x0a09080f0e0d0c0b))
+	return c40
+}
+
+func c48_DATA() Mem {
+	if c48_DATA_ptr != nil {
+		return *c48_DATA_ptr
+	}
+
+	c48 := GLOBL(ThatPeskyUnicodeDot+"c48", NOPTR|RODATA)
+	c48_DATA_ptr = &c48
+	DATA(0x00, U64(0x0100070605040302))
+	DATA(0x08, U64(0x09080f0e0d0c0b0a))
+	return c48
+}

blake2b/_asm/standard/go.mod

@@ -0,0 +1,15 @@
+module blake2b/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

blake2b/_asm/standard/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

blake2s/_asm/blake2s_amd64_asm.go

@@ -0,0 +1,525 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	"github.com/mmcloughlin/avo/ir"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/blake2s"
+)
+
+//go:generate go run . -out ../blake2s_amd64.s -pkg blake2s
+
+func main() {
+	Package("golang.org/x/crypto/blake2s")
+	ConstraintExpr("amd64,gc,!purego")
+	hashBlocksSSE2()
+	hashBlocksSSSE3()
+	hashBlocksSSE4()
+	Generate()
+}
+
+func ROTL_SSE2(n uint64, t, v VecPhysical) {
+	MOVO(v, t)
+	PSLLL(Imm(n), t)
+	PSRLL(Imm(32-n), v)
+	PXOR(t, v)
+}
+
+func ROTL_SSSE3(c, v VecPhysical) {
+	PSHUFB(c, v)
+}
+
+func ROUND_SSE2(v0, v1, v2, v3 VecPhysical, m0, m1, m2, m3 Mem, t VecPhysical) {
+	PADDL(m0, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(16, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m1, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(24, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v1, v1)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v3, v3)
+	PADDL(m2, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(16, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m3, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(24, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v3, v3)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v1, v1)
+}
+
+func ROUND_SSSE3(v0, v1, v2, v3 VecPhysical, m0, m1, m2, m3 Op, t, c16, c8 VecPhysical) {
+	PADDL(m0, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c16, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m1, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c8, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v1, v1)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v3, v3)
+	PADDL(m2, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c16, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m3, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c8, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v3, v3)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v1, v1)
+}
+
+func LOAD_MSG_SSE4(m0, m1, m2, m3 VecPhysical, src GPPhysical, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15 int) {
+	// Hack to get Avo to emit a MOVL instruction with a VecPhysical as the destination
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i0 * 4), m0}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i1*4), m0)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i2*4), m0)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i3*4), m0)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i4 * 4), m1}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i5*4), m1)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i6*4), m1)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i7*4), m1)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i8 * 4), m2}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i9*4), m2)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i10*4), m2)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i11*4), m2)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i12 * 4), m3}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i13*4), m3)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i14*4), m3)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i15*4), m3)
+}
+
+func PRECOMPUTE_MSG(dst GPPhysical, off int, src, R8, R9, R10, R11, R12, R13, R14, R15 GPPhysical) {
+	MOVQ(Mem{Base: src}.Offset(0*4), R8)
+	MOVQ(Mem{Base: src}.Offset(2*4), R9)
+	MOVQ(Mem{Base: src}.Offset(4*4), R10)
+	MOVQ(Mem{Base: src}.Offset(6*4), R11)
+	MOVQ(Mem{Base: src}.Offset(8*4), R12)
+	MOVQ(Mem{Base: src}.Offset(10*4), R13)
+	MOVQ(Mem{Base: src}.Offset(12*4), R14)
+	MOVQ(Mem{Base: src}.Offset(14*4), R15)
+
+	MOVL(R8L, Mem{Base: dst}.Offset(0*4+off+0))
+	MOVL(R8L, Mem{Base: dst}.Offset(9*4+off+64))
+	MOVL(R8L, Mem{Base: dst}.Offset(5*4+off+128))
+	MOVL(R8L, Mem{Base: dst}.Offset(14*4+off+192))
+	MOVL(R8L, Mem{Base: dst}.Offset(4*4+off+256))
+	MOVL(R8L, Mem{Base: dst}.Offset(2*4+off+320))
+	MOVL(R8L, Mem{Base: dst}.Offset(8*4+off+384))
+	MOVL(R8L, Mem{Base: dst}.Offset(12*4+off+448))
+	MOVL(R8L, Mem{Base: dst}.Offset(3*4+off+512))
+	MOVL(R8L, Mem{Base: dst}.Offset(15*4+off+576))
+	SHRQ(Imm(32), R8)
+	MOVL(R8L, Mem{Base: dst}.Offset(4*4+off+0))
+	MOVL(R8L, Mem{Base: dst}.Offset(8*4+off+64))
+	MOVL(R8L, Mem{Base: dst}.Offset(14*4+off+128))
+	MOVL(R8L, Mem{Base: dst}.Offset(5*4+off+192))
+	MOVL(R8L, Mem{Base: dst}.Offset(12*4+off+256))
+	MOVL(R8L, Mem{Base: dst}.Offset(11*4+off+320))
+	MOVL(R8L, Mem{Base: dst}.Offset(1*4+off+384))
+	MOVL(R8L, Mem{Base: dst}.Offset(6*4+off+448))
+	MOVL(R8L, Mem{Base: dst}.Offset(10*4+off+512))
+	MOVL(R8L, Mem{Base: dst}.Offset(3*4+off+576))
+
+	MOVL(R9L, Mem{Base: dst}.Offset(1*4+off+0))
+	MOVL(R9L, Mem{Base: dst}.Offset(13*4+off+64))
+	MOVL(R9L, Mem{Base: dst}.Offset(6*4+off+128))
+	MOVL(R9L, Mem{Base: dst}.Offset(8*4+off+192))
+	MOVL(R9L, Mem{Base: dst}.Offset(2*4+off+256))
+	MOVL(R9L, Mem{Base: dst}.Offset(0*4+off+320))
+	MOVL(R9L, Mem{Base: dst}.Offset(14*4+off+384))
+	MOVL(R9L, Mem{Base: dst}.Offset(11*4+off+448))
+	MOVL(R9L, Mem{Base: dst}.Offset(12*4+off+512))
+	MOVL(R9L, Mem{Base: dst}.Offset(4*4+off+576))
+	SHRQ(Imm(32), R9)
+	MOVL(R9L, Mem{Base: dst}.Offset(5*4+off+0))
+	MOVL(R9L, Mem{Base: dst}.Offset(15*4+off+64))
+	MOVL(R9L, Mem{Base: dst}.Offset(9*4+off+128))
+	MOVL(R9L, Mem{Base: dst}.Offset(1*4+off+192))
+	MOVL(R9L, Mem{Base: dst}.Offset(11*4+off+256))
+	MOVL(R9L, Mem{Base: dst}.Offset(7*4+off+320))
+	MOVL(R9L, Mem{Base: dst}.Offset(13*4+off+384))
+	MOVL(R9L, Mem{Base: dst}.Offset(3*4+off+448))
+	MOVL(R9L, Mem{Base: dst}.Offset(6*4+off+512))
+	MOVL(R9L, Mem{Base: dst}.Offset(10*4+off+576))
+
+	MOVL(R10L, Mem{Base: dst}.Offset(2*4+off+0))
+	MOVL(R10L, Mem{Base: dst}.Offset(1*4+off+64))
+	MOVL(R10L, Mem{Base: dst}.Offset(15*4+off+128))
+	MOVL(R10L, Mem{Base: dst}.Offset(10*4+off+192))
+	MOVL(R10L, Mem{Base: dst}.Offset(6*4+off+256))
+	MOVL(R10L, Mem{Base: dst}.Offset(8*4+off+320))
+	MOVL(R10L, Mem{Base: dst}.Offset(3*4+off+384))
+	MOVL(R10L, Mem{Base: dst}.Offset(13*4+off+448))
+	MOVL(R10L, Mem{Base: dst}.Offset(14*4+off+512))
+	MOVL(R10L, Mem{Base: dst}.Offset(5*4+off+576))
+	SHRQ(Imm(32), R10)
+	MOVL(R10L, Mem{Base: dst}.Offset(6*4+off+0))
+	MOVL(R10L, Mem{Base: dst}.Offset(11*4+off+64))
+	MOVL(R10L, Mem{Base: dst}.Offset(2*4+off+128))
+	MOVL(R10L, Mem{Base: dst}.Offset(9*4+off+192))
+	MOVL(R10L, Mem{Base: dst}.Offset(1*4+off+256))
+	MOVL(R10L, Mem{Base: dst}.Offset(13*4+off+320))
+	MOVL(R10L, Mem{Base: dst}.Offset(4*4+off+384))
+	MOVL(R10L, Mem{Base: dst}.Offset(8*4+off+448))
+	MOVL(R10L, Mem{Base: dst}.Offset(15*4+off+512))
+	MOVL(R10L, Mem{Base: dst}.Offset(7*4+off+576))
+
+	MOVL(R11L, Mem{Base: dst}.Offset(3*4+off+0))
+	MOVL(R11L, Mem{Base: dst}.Offset(7*4+off+64))
+	MOVL(R11L, Mem{Base: dst}.Offset(13*4+off+128))
+	MOVL(R11L, Mem{Base: dst}.Offset(12*4+off+192))
+	MOVL(R11L, Mem{Base: dst}.Offset(10*4+off+256))
+	MOVL(R11L, Mem{Base: dst}.Offset(1*4+off+320))
+	MOVL(R11L, Mem{Base: dst}.Offset(9*4+off+384))
+	MOVL(R11L, Mem{Base: dst}.Offset(14*4+off+448))
+	MOVL(R11L, Mem{Base: dst}.Offset(0*4+off+512))
+	MOVL(R11L, Mem{Base: dst}.Offset(6*4+off+576))
+	SHRQ(Imm(32), R11)
+	MOVL(R11L, Mem{Base: dst}.Offset(7*4+off+0))
+	MOVL(R11L, Mem{Base: dst}.Offset(14*4+off+64))
+	MOVL(R11L, Mem{Base: dst}.Offset(10*4+off+128))
+	MOVL(R11L, Mem{Base: dst}.Offset(0*4+off+192))
+	MOVL(R11L, Mem{Base: dst}.Offset(5*4+off+256))
+	MOVL(R11L, Mem{Base: dst}.Offset(9*4+off+320))
+	MOVL(R11L, Mem{Base: dst}.Offset(12*4+off+384))
+	MOVL(R11L, Mem{Base: dst}.Offset(1*4+off+448))
+	MOVL(R11L, Mem{Base: dst}.Offset(13*4+off+512))
+	MOVL(R11L, Mem{Base: dst}.Offset(2*4+off+576))
+
+	MOVL(R12L, Mem{Base: dst}.Offset(8*4+off+0))
+	MOVL(R12L, Mem{Base: dst}.Offset(5*4+off+64))
+	MOVL(R12L, Mem{Base: dst}.Offset(4*4+off+128))
+	MOVL(R12L, Mem{Base: dst}.Offset(15*4+off+192))
+	MOVL(R12L, Mem{Base: dst}.Offset(14*4+off+256))
+	MOVL(R12L, Mem{Base: dst}.Offset(3*4+off+320))
+	MOVL(R12L, Mem{Base: dst}.Offset(11*4+off+384))
+	MOVL(R12L, Mem{Base: dst}.Offset(10*4+off+448))
+	MOVL(R12L, Mem{Base: dst}.Offset(7*4+off+512))
+	MOVL(R12L, Mem{Base: dst}.Offset(1*4+off+576))
+	SHRQ(Imm(32), R12)
+	MOVL(R12L, Mem{Base: dst}.Offset(12*4+off+0))
+	MOVL(R12L, Mem{Base: dst}.Offset(2*4+off+64))
+	MOVL(R12L, Mem{Base: dst}.Offset(11*4+off+128))
+	MOVL(R12L, Mem{Base: dst}.Offset(4*4+off+192))
+	MOVL(R12L, Mem{Base: dst}.Offset(0*4+off+256))
+	MOVL(R12L, Mem{Base: dst}.Offset(15*4+off+320))
+	MOVL(R12L, Mem{Base: dst}.Offset(10*4+off+384))
+	MOVL(R12L, Mem{Base: dst}.Offset(7*4+off+448))
+	MOVL(R12L, Mem{Base: dst}.Offset(5*4+off+512))
+	MOVL(R12L, Mem{Base: dst}.Offset(9*4+off+576))
+
+	MOVL(R13L, Mem{Base: dst}.Offset(9*4+off+0))
+	MOVL(R13L, Mem{Base: dst}.Offset(4*4+off+64))
+	MOVL(R13L, Mem{Base: dst}.Offset(8*4+off+128))
+	MOVL(R13L, Mem{Base: dst}.Offset(13*4+off+192))
+	MOVL(R13L, Mem{Base: dst}.Offset(3*4+off+256))
+	MOVL(R13L, Mem{Base: dst}.Offset(5*4+off+320))
+	MOVL(R13L, Mem{Base: dst}.Offset(7*4+off+384))
+	MOVL(R13L, Mem{Base: dst}.Offset(15*4+off+448))
+	MOVL(R13L, Mem{Base: dst}.Offset(11*4+off+512))
+	MOVL(R13L, Mem{Base: dst}.Offset(0*4+off+576))
+	SHRQ(Imm(32), R13)
+	MOVL(R13L, Mem{Base: dst}.Offset(13*4+off+0))
+	MOVL(R13L, Mem{Base: dst}.Offset(10*4+off+64))
+	MOVL(R13L, Mem{Base: dst}.Offset(0*4+off+128))
+	MOVL(R13L, Mem{Base: dst}.Offset(3*4+off+192))
+	MOVL(R13L, Mem{Base: dst}.Offset(9*4+off+256))
+	MOVL(R13L, Mem{Base: dst}.Offset(6*4+off+320))
+	MOVL(R13L, Mem{Base: dst}.Offset(15*4+off+384))
+	MOVL(R13L, Mem{Base: dst}.Offset(4*4+off+448))
+	MOVL(R13L, Mem{Base: dst}.Offset(2*4+off+512))
+	MOVL(R13L, Mem{Base: dst}.Offset(12*4+off+576))
+
+	MOVL(R14L, Mem{Base: dst}.Offset(10*4+off+0))
+	MOVL(R14L, Mem{Base: dst}.Offset(12*4+off+64))
+	MOVL(R14L, Mem{Base: dst}.Offset(1*4+off+128))
+	MOVL(R14L, Mem{Base: dst}.Offset(6*4+off+192))
+	MOVL(R14L, Mem{Base: dst}.Offset(13*4+off+256))
+	MOVL(R14L, Mem{Base: dst}.Offset(4*4+off+320))
+	MOVL(R14L, Mem{Base: dst}.Offset(0*4+off+384))
+	MOVL(R14L, Mem{Base: dst}.Offset(2*4+off+448))
+	MOVL(R14L, Mem{Base: dst}.Offset(8*4+off+512))
+	MOVL(R14L, Mem{Base: dst}.Offset(14*4+off+576))
+	SHRQ(Imm(32), R14)
+	MOVL(R14L, Mem{Base: dst}.Offset(14*4+off+0))
+	MOVL(R14L, Mem{Base: dst}.Offset(3*4+off+64))
+	MOVL(R14L, Mem{Base: dst}.Offset(7*4+off+128))
+	MOVL(R14L, Mem{Base: dst}.Offset(2*4+off+192))
+	MOVL(R14L, Mem{Base: dst}.Offset(15*4+off+256))
+	MOVL(R14L, Mem{Base: dst}.Offset(12*4+off+320))
+	MOVL(R14L, Mem{Base: dst}.Offset(6*4+off+384))
+	MOVL(R14L, Mem{Base: dst}.Offset(0*4+off+448))
+	MOVL(R14L, Mem{Base: dst}.Offset(9*4+off+512))
+	MOVL(R14L, Mem{Base: dst}.Offset(11*4+off+576))
+
+	MOVL(R15L, Mem{Base: dst}.Offset(11*4+off+0))
+	MOVL(R15L, Mem{Base: dst}.Offset(0*4+off+64))
+	MOVL(R15L, Mem{Base: dst}.Offset(12*4+off+128))
+	MOVL(R15L, Mem{Base: dst}.Offset(7*4+off+192))
+	MOVL(R15L, Mem{Base: dst}.Offset(8*4+off+256))
+	MOVL(R15L, Mem{Base: dst}.Offset(14*4+off+320))
+	MOVL(R15L, Mem{Base: dst}.Offset(2*4+off+384))
+	MOVL(R15L, Mem{Base: dst}.Offset(5*4+off+448))
+	MOVL(R15L, Mem{Base: dst}.Offset(1*4+off+512))
+	MOVL(R15L, Mem{Base: dst}.Offset(13*4+off+576))
+	SHRQ(Imm(32), R15)
+	MOVL(R15L, Mem{Base: dst}.Offset(15*4+off+0))
+	MOVL(R15L, Mem{Base: dst}.Offset(6*4+off+64))
+	MOVL(R15L, Mem{Base: dst}.Offset(3*4+off+128))
+	MOVL(R15L, Mem{Base: dst}.Offset(11*4+off+192))
+	MOVL(R15L, Mem{Base: dst}.Offset(7*4+off+256))
+	MOVL(R15L, Mem{Base: dst}.Offset(10*4+off+320))
+	MOVL(R15L, Mem{Base: dst}.Offset(5*4+off+384))
+	MOVL(R15L, Mem{Base: dst}.Offset(9*4+off+448))
+	MOVL(R15L, Mem{Base: dst}.Offset(4*4+off+512))
+	MOVL(R15L, Mem{Base: dst}.Offset(8*4+off+576))
+}
+
+func BLAKE2s_SSE2() {
+	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15)
+	for i := 0; i < 10; i++ {
+		ROUND_SSE2(X4, X5, X6, X7, Mem{Base: BP}.Offset(16+64*i), Mem{Base: BP}.Offset(32+64*i), Mem{Base: BP}.Offset(48+64*i), Mem{Base: BP}.Offset(64+64*i), X8)
+	}
+}
+
+func BLAKE2s_SSSE3() {
+	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15)
+	for i := 0; i < 10; i++ {
+		ROUND_SSSE3(X4, X5, X6, X7, Mem{Base: BP}.Offset(16+64*i), Mem{Base: BP}.Offset(32+64*i), Mem{Base: BP}.Offset(48+64*i), Mem{Base: BP}.Offset(64+64*i), X8, X13, X14)
+	}
+}
+
+func BLAKE2s_SSE4() {
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+}
+
+func HASH_BLOCKS(h, c, flag, blocks_base, blocks_len Mem, BLAKE2s_FUNC func()) {
+	MOVQ(h, RAX)
+	MOVQ(c, RBX)
+	MOVL(flag, ECX)
+	MOVQ(blocks_base, RSI)
+	MOVQ(blocks_len, RDX)
+
+	MOVQ(RSP, RBP)
+	ADDQ(Imm(15), RBP)
+	ANDQ(I32(^15), RBP)
+
+	MOVQ(Mem{Base: BX}.Offset(0), R9)
+	MOVQ(R9, Mem{Base: BP}.Offset(0))
+	MOVQ(RCX, Mem{Base: BP}.Offset(8))
+
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: AX}.Offset(16), X1)
+
+	iv0 := iv0_DATA()
+	iv1 := iv1_DATA()
+	MOVOU(iv0, X2)
+	MOVOU(iv1, X3)
+
+	counter := counter_DATA()
+	rol16 := rol16_DATA()
+	rol8 := rol8_DATA()
+	MOVOU(counter, X12)
+	MOVOU(rol16, X13)
+	MOVOU(rol8, X14)
+	MOVO(Mem{Base: BP}.Offset(0), X15)
+
+	Label("loop")
+	MOVO(X0, X4)
+	MOVO(X1, X5)
+	MOVO(X2, X6)
+	MOVO(X3, X7)
+
+	PADDQ(X12, X15)
+	PXOR(X15, X7)
+
+	BLAKE2s_FUNC()
+
+	PXOR(X4, X0)
+	PXOR(X5, X1)
+	PXOR(X6, X0)
+	PXOR(X7, X1)
+
+	LEAQ(Mem{Base: SI}.Offset(64), RSI)
+	SUBQ(Imm(64), RDX)
+	JNE(LabelRef("loop"))
+
+	MOVO(X15, Mem{Base: BP}.Offset(0))
+	MOVQ(Mem{Base: BP}.Offset(0), R9)
+	MOVQ(R9, Mem{Base: BX}.Offset(0))
+
+	MOVOU(X0, Mem{Base: AX}.Offset(0))
+	MOVOU(X1, Mem{Base: AX}.Offset(16))
+}
+
+func hashBlocksSSE2() {
+	Implement("hashBlocksSSE2")
+	Attributes(0)
+	AllocLocal(672) // frame = 656 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSE2)
+	RET()
+}
+
+func hashBlocksSSSE3() {
+	Implement("hashBlocksSSSE3")
+	Attributes(0)
+	AllocLocal(672) // frame = 656 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSSE3)
+	RET()
+}
+
+func hashBlocksSSE4() {
+	Implement("hashBlocksSSE4")
+	Attributes(0)
+	AllocLocal(32) // frame = 16 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSE4)
+	RET()
+}
+
+// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
+
+var iv0_DATA_ptr, iv1_DATA_ptr, rol16_DATA_ptr, rol8_DATA_ptr, counter_DATA_ptr *Mem
+
+func iv0_DATA() Mem {
+	if iv0_DATA_ptr != nil {
+		return *iv0_DATA_ptr
+	}
+
+	iv0_DATA := GLOBL("iv0", NOPTR|RODATA)
+	iv0_DATA_ptr = &iv0_DATA
+	DATA(0x00, U32(0x6a09e667))
+	DATA(0x04, U32(0xbb67ae85))
+	DATA(0x08, U32(0x3c6ef372))
+	DATA(0x0c, U32(0xa54ff53a))
+	return iv0_DATA
+}
+
+func iv1_DATA() Mem {
+	if iv1_DATA_ptr != nil {
+		return *iv1_DATA_ptr
+	}
+
+	iv1_DATA := GLOBL("iv1", NOPTR|RODATA)
+	iv1_DATA_ptr = &iv1_DATA
+	DATA(0x00, U32(0x510e527f))
+	DATA(0x04, U32(0x9b05688c))
+	DATA(0x08, U32(0x1f83d9ab))
+	DATA(0x0c, U32(0x5be0cd19))
+	return iv1_DATA
+}
+
+func rol16_DATA() Mem {
+	if rol16_DATA_ptr != nil {
+		return *rol16_DATA_ptr
+	}
+
+	rol16_DATA := GLOBL("rol16", NOPTR|RODATA)
+	rol16_DATA_ptr = &rol16_DATA
+	DATA(0x00, U64(0x0504070601000302))
+	DATA(0x08, U64(0x0D0C0F0E09080B0A))
+	return rol16_DATA
+}
+
+func rol8_DATA() Mem {
+	if rol8_DATA_ptr != nil {
+		return *rol8_DATA_ptr
+	}
+
+	rol8_DATA := GLOBL("rol8", NOPTR|RODATA)
+	rol8_DATA_ptr = &rol8_DATA
+	DATA(0x00, U64(0x0407060500030201))
+	DATA(0x08, U64(0x0C0F0E0D080B0A09))
+	return rol8_DATA
+}
+
+func counter_DATA() Mem {
+	if counter_DATA_ptr != nil {
+		return *counter_DATA_ptr
+	}
+
+	counter_DATA := GLOBL("counter", NOPTR|RODATA)
+	counter_DATA_ptr = &counter_DATA
+	DATA(0x00, U64(0x0000000000000040))
+	DATA(0x08, U64(0x0000000000000000))
+	return counter_DATA
+}

blake2s/_asm/go.mod

@@ -0,0 +1,15 @@
+module blake2s/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

blake2s/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

bn256/gfp12.go

@@ -5,7 +5,7 @@
 package bn256
 
 // For details of the algorithms used, see "Multiplication and Squaring on
-// Pairing-Friendly Fields, Devegili et al.
+// Pairing-Friendly Fields", Devegili et al.
 // http://eprint.iacr.org/2006/471.pdf.
 
 import (

bn256/gfp2.go

@@ -5,7 +5,7 @@
 package bn256
 
 // For details of the algorithms used, see "Multiplication and Squaring on
-// Pairing-Friendly Fields, Devegili et al.
+// Pairing-Friendly Fields", Devegili et al.
 // http://eprint.iacr.org/2006/471.pdf.
 
 import (

bn256/gfp6.go

@@ -5,7 +5,7 @@
 package bn256
 
 // For details of the algorithms used, see "Multiplication and Squaring on
-// Pairing-Friendly Fields, Devegili et al.
+// Pairing-Friendly Fields", Devegili et al.
 // http://eprint.iacr.org/2006/471.pdf.
 
 import (

chacha20poly1305/_asm/go.mod

@@ -0,0 +1,15 @@
+module chacha20poly1305/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

chacha20poly1305/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

go.mod

@@ -4,8 +4,8 @@ go 1.20
 
 require (
 	golang.org/x/net v0.21.0 // tagx:ignore
-	golang.org/x/sys v0.23.0
-	golang.org/x/term v0.23.0
+	golang.org/x/sys v0.26.0
+	golang.org/x/term v0.25.0
 )
 
-require golang.org/x/text v0.17.0 // indirect
+require golang.org/x/text v0.19.0 // indirect

go.sum

@@ -1,8 +1,8 @@
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
-golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
-golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
+golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=

internal/poly1305/_asm/go.mod

@@ -0,0 +1,15 @@
+module internal/poly1305/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

internal/poly1305/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

internal/poly1305/_asm/sum_amd64_asm.go

@@ -0,0 +1,126 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/sha3"
+)
+
+//go:generate go run . -out ../sum_amd64.s -pkg poly1305
+
+func main() {
+	Package("golang.org/x/crypto/internal/poly1305")
+	ConstraintExpr("gc,!purego")
+	update()
+	Generate()
+}
+
+func update() {
+	Implement("update")
+
+	Load(Param("state"), RDI)
+	MOVQ(NewParamAddr("msg_base", 8), RSI)
+	MOVQ(NewParamAddr("msg_len", 16), R15)
+
+	MOVQ(Mem{Base: DI}.Offset(0), R8)   // h0
+	MOVQ(Mem{Base: DI}.Offset(8), R9)   // h1
+	MOVQ(Mem{Base: DI}.Offset(16), R10) // h2
+	MOVQ(Mem{Base: DI}.Offset(24), R11) // r0
+	MOVQ(Mem{Base: DI}.Offset(32), R12) // r1
+
+	CMPQ(R15, Imm(16))
+	JB(LabelRef("bytes_between_0_and_15"))
+
+	Label("loop")
+	POLY1305_ADD(RSI, R8, R9, R10)
+
+	Label("multiply")
+	POLY1305_MUL(R8, R9, R10, R11, R12, RBX, RCX, R13, R14)
+	SUBQ(Imm(16), R15)
+	CMPQ(R15, Imm(16))
+	JAE(LabelRef("loop"))
+
+	Label("bytes_between_0_and_15")
+	TESTQ(R15, R15)
+	JZ(LabelRef("done"))
+	MOVQ(U32(1), RBX)
+	XORQ(RCX, RCX)
+	XORQ(R13, R13)
+	ADDQ(R15, RSI)
+
+	Label("flush_buffer")
+	SHLQ(Imm(8), RBX, RCX)
+	SHLQ(Imm(8), RBX)
+	MOVB(Mem{Base: SI}.Offset(-1), R13B)
+	XORQ(R13, RBX)
+	DECQ(RSI)
+	DECQ(R15)
+	JNZ(LabelRef("flush_buffer"))
+
+	ADDQ(RBX, R8)
+	ADCQ(RCX, R9)
+	ADCQ(Imm(0), R10)
+	MOVQ(U32(16), R15)
+	JMP(LabelRef("multiply"))
+
+	Label("done")
+	MOVQ(R8, Mem{Base: DI}.Offset(0))
+	MOVQ(R9, Mem{Base: DI}.Offset(8))
+	MOVQ(R10, Mem{Base: DI}.Offset(16))
+	RET()
+}
+
+func POLY1305_ADD(msg, h0, h1, h2 GPPhysical) {
+	ADDQ(Mem{Base: msg}.Offset(0), h0)
+	ADCQ(Mem{Base: msg}.Offset(8), h1)
+	ADCQ(Imm(1), h2)
+	LEAQ(Mem{Base: msg}.Offset(16), msg)
+}
+
+func POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3 GPPhysical) {
+	MOVQ(r0, RAX)
+	MULQ(h0)
+	MOVQ(RAX, t0)
+	MOVQ(RDX, t1)
+	MOVQ(r0, RAX)
+	MULQ(h1)
+	ADDQ(RAX, t1)
+	ADCQ(Imm(0), RDX)
+	MOVQ(r0, t2)
+	IMULQ(h2, t2)
+	ADDQ(RDX, t2)
+
+	MOVQ(r1, RAX)
+	MULQ(h0)
+	ADDQ(RAX, t1)
+	ADCQ(Imm(0), RDX)
+	MOVQ(RDX, h0)
+	MOVQ(r1, t3)
+	IMULQ(h2, t3)
+	MOVQ(r1, RAX)
+	MULQ(h1)
+	ADDQ(RAX, t2)
+	ADCQ(RDX, t3)
+	ADDQ(h0, t2)
+	ADCQ(Imm(0), t3)
+
+	MOVQ(t0, h0)
+	MOVQ(t1, h1)
+	MOVQ(t2, h2)
+	ANDQ(Imm(3), h2)
+	MOVQ(t2, t0)
+	ANDQ(I32(-4), t0)
+	ADDQ(t0, h0)
+	ADCQ(t3, h1)
+	ADCQ(Imm(0), h2)
+	SHRQ(Imm(2), t3, t2)
+	SHRQ(Imm(2), t3)
+	ADDQ(t2, h0)
+	ADCQ(t3, h1)
+	ADCQ(Imm(0), h2)
+}

internal/poly1305/sum_amd64.s

@@ -1,108 +1,93 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run sum_amd64_asm.go -out ../sum_amd64.s -pkg poly1305. DO NOT EDIT.
 
 //go:build gc && !purego
 
-#include "textflag.h"
-
-#define POLY1305_ADD(msg, h0, h1, h2) \
-	ADDQ 0(msg), h0;  \
-	ADCQ 8(msg), h1;  \
-	ADCQ $1, h2;      \
-	LEAQ 16(msg), msg
-
-#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \
-	MOVQ  r0, AX;                  \
-	MULQ  h0;                      \
-	MOVQ  AX, t0;                  \
-	MOVQ  DX, t1;                  \
-	MOVQ  r0, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  r0, t2;                  \
-	IMULQ h2, t2;                  \
-	ADDQ  DX, t2;                  \
-	                               \
-	MOVQ  r1, AX;                  \
-	MULQ  h0;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  DX, h0;                  \
-	MOVQ  r1, t3;                  \
-	IMULQ h2, t3;                  \
-	MOVQ  r1, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t2;                  \
-	ADCQ  DX, t3;                  \
-	ADDQ  h0, t2;                  \
-	ADCQ  $0, t3;                  \
-	                               \
-	MOVQ  t0, h0;                  \
-	MOVQ  t1, h1;                  \
-	MOVQ  t2, h2;                  \
-	ANDQ  $3, h2;                  \
-	MOVQ  t2, t0;                  \
-	ANDQ  $0xFFFFFFFFFFFFFFFC, t0; \
-	ADDQ  t0, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2;                  \
-	SHRQ  $2, t3, t2;              \
-	SHRQ  $2, t3;                  \
-	ADDQ  t2, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2
-
-// func update(state *[7]uint64, msg []byte)
+// func update(state *macState, msg []byte)
 TEXT ·update(SB), $0-32
 	MOVQ state+0(FP), DI
 	MOVQ msg_base+8(FP), SI
 	MOVQ msg_len+16(FP), R15
-
-	MOVQ 0(DI), R8   // h0
-	MOVQ 8(DI), R9   // h1
-	MOVQ 16(DI), R10 // h2
-	MOVQ 24(DI), R11 // r0
-	MOVQ 32(DI), R12 // r1
-
-	CMPQ R15, $16
+	MOVQ (DI), R8
+	MOVQ 8(DI), R9
+	MOVQ 16(DI), R10
+	MOVQ 24(DI), R11
+	MOVQ 32(DI), R12
+	CMPQ R15, $0x10
 	JB   bytes_between_0_and_15
 
 loop:
-	POLY1305_ADD(SI, R8, R9, R10)
+	ADDQ (SI), R8
+	ADCQ 8(SI), R9
+	ADCQ $0x01, R10
+	LEAQ 16(SI), SI
 
 multiply:
-	POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14)
-	SUBQ $16, R15
-	CMPQ R15, $16
-	JAE  loop
+	MOVQ  R11, AX
+	MULQ  R8
+	MOVQ  AX, BX
+	MOVQ  DX, CX
+	MOVQ  R11, AX
+	MULQ  R9
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  R11, R13
+	IMULQ R10, R13
+	ADDQ  DX, R13
+	MOVQ  R12, AX
+	MULQ  R8
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  DX, R8
+	MOVQ  R12, R14
+	IMULQ R10, R14
+	MOVQ  R12, AX
+	MULQ  R9
+	ADDQ  AX, R13
+	ADCQ  DX, R14
+	ADDQ  R8, R13
+	ADCQ  $0x00, R14
+	MOVQ  BX, R8
+	MOVQ  CX, R9
+	MOVQ  R13, R10
+	ANDQ  $0x03, R10
+	MOVQ  R13, BX
+	ANDQ  $-4, BX
+	ADDQ  BX, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SHRQ  $0x02, R14, R13
+	SHRQ  $0x02, R14
+	ADDQ  R13, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SUBQ  $0x10, R15
+	CMPQ  R15, $0x10
+	JAE   loop
 
 bytes_between_0_and_15:
 	TESTQ R15, R15
 	JZ    done
-	MOVQ  $1, BX
+	MOVQ  $0x00000001, BX
 	XORQ  CX, CX
 	XORQ  R13, R13
 	ADDQ  R15, SI
 
 flush_buffer:
-	SHLQ $8, BX, CX
-	SHLQ $8, BX
+	SHLQ $0x08, BX, CX
+	SHLQ $0x08, BX
 	MOVB -1(SI), R13
 	XORQ R13, BX
 	DECQ SI
 	DECQ R15
 	JNZ  flush_buffer
-
 	ADDQ BX, R8
 	ADCQ CX, R9
-	ADCQ $0, R10
-	MOVQ $16, R15
+	ADCQ $0x00, R10
+	MOVQ $0x00000010, R15
 	JMP  multiply
 
 done:
-	MOVQ R8, 0(DI)
+	MOVQ R8, (DI)
 	MOVQ R9, 8(DI)
 	MOVQ R10, 16(DI)
 	RET

salsa20/salsa/_asm/go.mod

@@ -0,0 +1,14 @@
+module salsa20/salsa/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

salsa20/salsa/_asm/go.sum

@@ -0,0 +1,10 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

salsa20/salsa/_asm/salsa20_amd64_asm.go

@@ -0,0 +1,932 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	"github.com/mmcloughlin/avo/ir"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/salsa20/salsa"
+)
+
+//go:generate go run . -out ../salsa20_amd64.s -pkg salsa
+
+func main() {
+	Package("golang.org/x/crypto/salsa20/salsa")
+	ConstraintExpr("amd64,!purego,gc")
+	salsa2020XORKeyStream()
+	Generate()
+}
+
+func salsa2020XORKeyStream() {
+	Implement("salsa2020XORKeyStream")
+	Attributes(0)
+	AllocLocal(456) // frame = 424 + 32 byte alignment
+	Comment("This needs up to 64 bytes at 360(R12); hence the non-obvious frame size.")
+
+	Load(Param("out"), RDI)
+	Load(Param("in"), RSI)
+	Load(Param("n"), RDX)
+	Load(Param("nonce"), RCX)
+	Load(Param("key"), R8)
+
+	MOVQ(RSP, R12)
+	ADDQ(Imm(31), R12)
+	ANDQ(I32(^31), R12)
+
+	MOVQ(RDX, R9)
+	MOVQ(RCX, RDX)
+	MOVQ(R8, R10)
+	CMPQ(R9, Imm(0))
+	JBE(LabelRef("DONE"))
+
+	START()
+	BYTESATLEAST256()
+	MAINLOOP1()
+	BYTESBETWEEN1AND255()
+	NOCOPY()
+	MAINLOOP2()
+
+	Label("BYTESATLEAST64")
+	Label("DONE")
+	RET()
+	Label("BYTESATLEAST65")
+	SUBQ(Imm(64), R9)
+	ADDQ(Imm(64), RDI)
+	ADDQ(Imm(64), RSI)
+	JMP(LabelRef("BYTESBETWEEN1AND255"))
+}
+
+func START() {
+	Label("START")
+	MOVL(Mem{Base: R10}.Offset(20), ECX)
+	MOVL(Mem{Base: R10}.Offset(0), R8L)
+	MOVL(Mem{Base: EDX}.Offset(0), EAX)
+	MOVL(Mem{Base: R10}.Offset(16), R11L)
+	MOVL(ECX, Mem{Base: R12}.Offset(0))
+	MOVL(R8L, Mem{Base: R12}.Offset(4))
+	MOVL(EAX, Mem{Base: R12}.Offset(8))
+	MOVL(R11L, Mem{Base: R12}.Offset(12))
+	MOVL(Mem{Base: EDX}.Offset(8), ECX)
+	MOVL(Mem{Base: R10}.Offset(24), R8L)
+	MOVL(Mem{Base: R10}.Offset(4), EAX)
+	MOVL(Mem{Base: EDX}.Offset(4), R11L)
+	MOVL(ECX, Mem{Base: R12}.Offset(16))
+	MOVL(R8L, Mem{Base: R12}.Offset(20))
+	MOVL(EAX, Mem{Base: R12}.Offset(24))
+	MOVL(R11L, Mem{Base: R12}.Offset(28))
+	MOVL(Mem{Base: EDX}.Offset(12), ECX)
+	MOVL(Mem{Base: R10}.Offset(12), EDX)
+	MOVL(Mem{Base: R10}.Offset(28), R8L)
+	MOVL(Mem{Base: R10}.Offset(8), EAX)
+	MOVL(EDX, Mem{Base: R12}.Offset(32))
+	MOVL(ECX, Mem{Base: R12}.Offset(36))
+	MOVL(R8L, Mem{Base: R12}.Offset(40))
+	MOVL(EAX, Mem{Base: R12}.Offset(44))
+	MOVQ(Imm(1634760805), RDX)
+	MOVQ(Imm(857760878), RCX)
+	MOVQ(Imm(2036477234), R8)
+	MOVQ(Imm(1797285236), RAX)
+	MOVL(EDX, Mem{Base: R12}.Offset(48))
+	MOVL(ECX, Mem{Base: R12}.Offset(52))
+	MOVL(R8L, Mem{Base: R12}.Offset(56))
+	MOVL(EAX, Mem{Base: R12}.Offset(60))
+	CMPQ(R9, U32(256))
+	JB(LabelRef("BYTESBETWEEN1AND255"))
+	MOVOA(Mem{Base: R12}.Offset(48), X0)
+	PSHUFL(Imm(0x55), X0, X1)
+	PSHUFL(Imm(0xAA), X0, X2)
+	PSHUFL(Imm(0xFF), X0, X3)
+	PSHUFL(Imm(0x00), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(64))
+	MOVOA(X2, Mem{Base: R12}.Offset(80))
+	MOVOA(X3, Mem{Base: R12}.Offset(96))
+	MOVOA(X0, Mem{Base: R12}.Offset(112))
+	MOVOA(Mem{Base: R12}.Offset(0), X0)
+	PSHUFL(Imm(0xAA), X0, X1)
+	PSHUFL(Imm(0xFF), X0, X2)
+	PSHUFL(Imm(0x00), X0, X3)
+	PSHUFL(Imm(0x55), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(128))
+	MOVOA(X2, Mem{Base: R12}.Offset(144))
+	MOVOA(X3, Mem{Base: R12}.Offset(160))
+	MOVOA(X0, Mem{Base: R12}.Offset(176))
+	MOVOA(Mem{Base: R12}.Offset(16), X0)
+	PSHUFL(Imm(0xFF), X0, X1)
+	PSHUFL(Imm(0x55), X0, X2)
+	PSHUFL(Imm(0xAA), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(192))
+	MOVOA(X2, Mem{Base: R12}.Offset(208))
+	MOVOA(X0, Mem{Base: R12}.Offset(224))
+	MOVOA(Mem{Base: R12}.Offset(32), X0)
+	PSHUFL(Imm(0x00), X0, X1)
+	PSHUFL(Imm(0xAA), X0, X2)
+	PSHUFL(Imm(0xFF), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(240))
+	MOVOA(X2, Mem{Base: R12}.Offset(256))
+	MOVOA(X0, Mem{Base: R12}.Offset(272))
+
+}
+
+func BYTESATLEAST256() {
+	Label("BYTESATLEAST256")
+	MOVL(Mem{Base: R12}.Offset(16), EDX)
+	MOVL(Mem{Base: R12}.Offset(36), ECX)
+	MOVL(EDX, Mem{Base: R12}.Offset(288))
+	MOVL(ECX, Mem{Base: R12}.Offset(304))
+	SHLQ(Imm(32), RCX)
+	ADDQ(RCX, RDX)
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(292))
+	MOVL(ECX, Mem{Base: R12}.Offset(308))
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(296))
+	MOVL(ECX, Mem{Base: R12}.Offset(312))
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(300))
+	MOVL(ECX, Mem{Base: R12}.Offset(316))
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(16))
+	MOVL(ECX, Mem{Base: R12}.Offset(36))
+	MOVQ(R9, Mem{Base: R12}.Offset(352))
+	MOVQ(U32(20), RDX)
+	MOVOA(Mem{Base: R12}.Offset(64), X0)
+	MOVOA(Mem{Base: R12}.Offset(80), X1)
+	MOVOA(Mem{Base: R12}.Offset(96), X2)
+	MOVOA(Mem{Base: R12}.Offset(256), X3)
+	MOVOA(Mem{Base: R12}.Offset(272), X4)
+	MOVOA(Mem{Base: R12}.Offset(128), X5)
+	MOVOA(Mem{Base: R12}.Offset(144), X6)
+	MOVOA(Mem{Base: R12}.Offset(176), X7)
+	MOVOA(Mem{Base: R12}.Offset(192), X8)
+	MOVOA(Mem{Base: R12}.Offset(208), X9)
+	MOVOA(Mem{Base: R12}.Offset(224), X10)
+	MOVOA(Mem{Base: R12}.Offset(304), X11)
+	MOVOA(Mem{Base: R12}.Offset(112), X12)
+	MOVOA(Mem{Base: R12}.Offset(160), X13)
+	MOVOA(Mem{Base: R12}.Offset(240), X14)
+	MOVOA(Mem{Base: R12}.Offset(288), X15)
+}
+
+func MAINLOOP1() {
+	Label("MAINLOOP1")
+	MOVOA(X1, Mem{Base: R12}.Offset(320))
+	MOVOA(X2, Mem{Base: R12}.Offset(336))
+	MOVOA(X13, X1)
+	PADDL(X12, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X14)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X14)
+	MOVOA(X7, X1)
+	PADDL(X0, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X11)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X11)
+	MOVOA(X12, X1)
+	PADDL(X14, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X15)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X15)
+	MOVOA(X0, X1)
+	PADDL(X11, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X9)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X9)
+	MOVOA(X14, X1)
+	PADDL(X15, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X13)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X13)
+	MOVOA(X11, X1)
+	PADDL(X9, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X7)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X7)
+	MOVOA(X15, X1)
+	PADDL(X13, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(18), X1)
+	PXOR(X1, X12)
+	PSRLL(Imm(14), X2)
+	PXOR(X2, X12)
+	MOVOA(Mem{Base: R12}.Offset(320), X1)
+	MOVOA(X12, Mem{Base: R12}.Offset(320))
+	MOVOA(X9, X2)
+	PADDL(X7, X2)
+	MOVOA(X2, X12)
+	PSLLL(Imm(18), X2)
+	PXOR(X2, X0)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X0)
+	MOVOA(X5, X2)
+	PADDL(X1, X2)
+	MOVOA(X2, X12)
+	PSLLL(Imm(7), X2)
+	PXOR(X2, X3)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X3)
+	MOVOA(Mem{Base: R12}.Offset(336), X2)
+	MOVOA(X0, Mem{Base: R12}.Offset(336))
+	MOVOA(X6, X0)
+	PADDL(X2, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(7), X0)
+	PXOR(X0, X4)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X4)
+	MOVOA(X1, X0)
+	PADDL(X3, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X10)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X10)
+	MOVOA(X2, X0)
+	PADDL(X4, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X8)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X8)
+	MOVOA(X3, X0)
+	PADDL(X10, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X5)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X5)
+	MOVOA(X4, X0)
+	PADDL(X8, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X6)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X6)
+	MOVOA(X10, X0)
+	PADDL(X5, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X1)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X1)
+	MOVOA(Mem{Base: R12}.Offset(320), X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(320))
+	MOVOA(X4, X1)
+	PADDL(X0, X1)
+	MOVOA(X1, X12)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X7)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X7)
+	MOVOA(X8, X1)
+	PADDL(X6, X1)
+	MOVOA(X1, X12)
+	PSLLL(Imm(18), X1)
+	PXOR(X1, X2)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X2)
+	MOVOA(Mem{Base: R12}.Offset(336), X12)
+	MOVOA(X2, Mem{Base: R12}.Offset(336))
+	MOVOA(X14, X1)
+	PADDL(X12, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X5)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X5)
+	MOVOA(X0, X1)
+	PADDL(X7, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X10)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X10)
+	MOVOA(X12, X1)
+	PADDL(X5, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X8)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X8)
+	MOVOA(X7, X1)
+	PADDL(X10, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X4)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X4)
+	MOVOA(X5, X1)
+	PADDL(X8, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X14)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X14)
+	MOVOA(X10, X1)
+	PADDL(X4, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(18), X1)
+	PXOR(X1, X0)
+	PSRLL(Imm(14), X2)
+	PXOR(X2, X0)
+	MOVOA(Mem{Base: R12}.Offset(320), X1)
+	MOVOA(X0, Mem{Base: R12}.Offset(320))
+	MOVOA(X8, X0)
+	PADDL(X14, X0)
+	MOVOA(X0, X2)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X12)
+	PSRLL(Imm(14), X2)
+	PXOR(X2, X12)
+	MOVOA(X11, X0)
+	PADDL(X1, X0)
+	MOVOA(X0, X2)
+	PSLLL(Imm(7), X0)
+	PXOR(X0, X6)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X6)
+	MOVOA(Mem{Base: R12}.Offset(336), X2)
+	MOVOA(X12, Mem{Base: R12}.Offset(336))
+	MOVOA(X3, X0)
+	PADDL(X2, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(7), X0)
+	PXOR(X0, X13)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X13)
+	MOVOA(X1, X0)
+	PADDL(X6, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X15)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X15)
+	MOVOA(X2, X0)
+	PADDL(X13, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X9)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X9)
+	MOVOA(X6, X0)
+	PADDL(X15, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X11)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X11)
+	MOVOA(X13, X0)
+	PADDL(X9, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X3)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X3)
+	MOVOA(X15, X0)
+	PADDL(X11, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X1)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X1)
+	MOVOA(X9, X0)
+	PADDL(X3, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X2)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X2)
+	MOVOA(Mem{Base: R12}.Offset(320), X12)
+	MOVOA(Mem{Base: R12}.Offset(336), X0)
+	SUBQ(Imm(2), RDX)
+	JA(LabelRef("MAINLOOP1"))
+	PADDL(Mem{Base: R12}.Offset(112), X12)
+	PADDL(Mem{Base: R12}.Offset(176), X7)
+	PADDL(Mem{Base: R12}.Offset(224), X10)
+	PADDL(Mem{Base: R12}.Offset(272), X4)
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	PSHUFL(Imm(0x39), X12, X12)
+	PSHUFL(Imm(0x39), X7, X7)
+	PSHUFL(Imm(0x39), X10, X10)
+	PSHUFL(Imm(0x39), X4, X4)
+	XORL(Mem{Base: SI}.Offset(0), EDX)
+	XORL(Mem{Base: SI}.Offset(4), ECX)
+	XORL(Mem{Base: SI}.Offset(8), R8L)
+	XORL(Mem{Base: SI}.Offset(12), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(0))
+	MOVL(ECX, Mem{Base: DI}.Offset(4))
+	MOVL(R8L, Mem{Base: DI}.Offset(8))
+	MOVL(R9L, Mem{Base: DI}.Offset(12))
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	PSHUFL(Imm(0x39), X12, X12)
+	PSHUFL(Imm(0x39), X7, X7)
+	PSHUFL(Imm(0x39), X10, X10)
+	PSHUFL(Imm(0x39), X4, X4)
+	XORL(Mem{Base: SI}.Offset(64), EDX)
+	XORL(Mem{Base: SI}.Offset(68), ECX)
+	XORL(Mem{Base: SI}.Offset(72), R8L)
+	XORL(Mem{Base: SI}.Offset(76), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(64))
+	MOVL(ECX, Mem{Base: DI}.Offset(68))
+	MOVL(R8L, Mem{Base: DI}.Offset(72))
+	MOVL(R9L, Mem{Base: DI}.Offset(76))
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	PSHUFL(Imm(0x39), X12, X12)
+	PSHUFL(Imm(0x39), X7, X7)
+	PSHUFL(Imm(0x39), X10, X10)
+	PSHUFL(Imm(0x39), X4, X4)
+	XORL(Mem{Base: SI}.Offset(128), EDX)
+	XORL(Mem{Base: SI}.Offset(132), ECX)
+	XORL(Mem{Base: SI}.Offset(136), R8L)
+	XORL(Mem{Base: SI}.Offset(140), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(128))
+	MOVL(ECX, Mem{Base: DI}.Offset(132))
+	MOVL(R8L, Mem{Base: DI}.Offset(136))
+	MOVL(R9L, Mem{Base: DI}.Offset(140))
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	XORL(Mem{Base: SI}.Offset(192), EDX)
+	XORL(Mem{Base: SI}.Offset(196), ECX)
+	XORL(Mem{Base: SI}.Offset(200), R8L)
+	XORL(Mem{Base: SI}.Offset(204), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(192))
+	MOVL(ECX, Mem{Base: DI}.Offset(196))
+	MOVL(R8L, Mem{Base: DI}.Offset(200))
+	MOVL(R9L, Mem{Base: DI}.Offset(204))
+	PADDL(Mem{Base: R12}.Offset(240), X14)
+	PADDL(Mem{Base: R12}.Offset(64), X0)
+	PADDL(Mem{Base: R12}.Offset(128), X5)
+	PADDL(Mem{Base: R12}.Offset(192), X8)
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	PSHUFL(Imm(0x39), X14, X14)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X5, X5)
+	PSHUFL(Imm(0x39), X8, X8)
+	XORL(Mem{Base: SI}.Offset(16), EDX)
+	XORL(Mem{Base: SI}.Offset(20), ECX)
+	XORL(Mem{Base: SI}.Offset(24), R8L)
+	XORL(Mem{Base: SI}.Offset(28), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(16))
+	MOVL(ECX, Mem{Base: DI}.Offset(20))
+	MOVL(R8L, Mem{Base: DI}.Offset(24))
+	MOVL(R9L, Mem{Base: DI}.Offset(28))
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	PSHUFL(Imm(0x39), X14, X14)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X5, X5)
+	PSHUFL(Imm(0x39), X8, X8)
+	XORL(Mem{Base: SI}.Offset(80), EDX)
+	XORL(Mem{Base: SI}.Offset(84), ECX)
+	XORL(Mem{Base: SI}.Offset(88), R8L)
+	XORL(Mem{Base: SI}.Offset(92), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(80))
+	MOVL(ECX, Mem{Base: DI}.Offset(84))
+	MOVL(R8L, Mem{Base: DI}.Offset(88))
+	MOVL(R9L, Mem{Base: DI}.Offset(92))
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	PSHUFL(Imm(0x39), X14, X14)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X5, X5)
+	PSHUFL(Imm(0x39), X8, X8)
+	XORL(Mem{Base: SI}.Offset(144), EDX)
+	XORL(Mem{Base: SI}.Offset(148), ECX)
+	XORL(Mem{Base: SI}.Offset(152), R8L)
+	XORL(Mem{Base: SI}.Offset(156), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(144))
+	MOVL(ECX, Mem{Base: DI}.Offset(148))
+	MOVL(R8L, Mem{Base: DI}.Offset(152))
+	MOVL(R9L, Mem{Base: DI}.Offset(156))
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	XORL(Mem{Base: SI}.Offset(208), EDX)
+	XORL(Mem{Base: SI}.Offset(212), ECX)
+	XORL(Mem{Base: SI}.Offset(216), R8L)
+	XORL(Mem{Base: SI}.Offset(220), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(208))
+	MOVL(ECX, Mem{Base: DI}.Offset(212))
+	MOVL(R8L, Mem{Base: DI}.Offset(216))
+	MOVL(R9L, Mem{Base: DI}.Offset(220))
+	PADDL(Mem{Base: R12}.Offset(288), X15)
+	PADDL(Mem{Base: R12}.Offset(304), X11)
+	PADDL(Mem{Base: R12}.Offset(80), X1)
+	PADDL(Mem{Base: R12}.Offset(144), X6)
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	PSHUFL(Imm(0x39), X15, X15)
+	PSHUFL(Imm(0x39), X11, X11)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X6, X6)
+	XORL(Mem{Base: SI}.Offset(32), EDX)
+	XORL(Mem{Base: SI}.Offset(36), ECX)
+	XORL(Mem{Base: SI}.Offset(40), R8L)
+	XORL(Mem{Base: SI}.Offset(44), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(32))
+	MOVL(ECX, Mem{Base: DI}.Offset(36))
+	MOVL(R8L, Mem{Base: DI}.Offset(40))
+	MOVL(R9L, Mem{Base: DI}.Offset(44))
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	PSHUFL(Imm(0x39), X15, X15)
+	PSHUFL(Imm(0x39), X11, X11)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X6, X6)
+	XORL(Mem{Base: SI}.Offset(96), EDX)
+	XORL(Mem{Base: SI}.Offset(100), ECX)
+	XORL(Mem{Base: SI}.Offset(104), R8L)
+	XORL(Mem{Base: SI}.Offset(108), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(96))
+	MOVL(ECX, Mem{Base: DI}.Offset(100))
+	MOVL(R8L, Mem{Base: DI}.Offset(104))
+	MOVL(R9L, Mem{Base: DI}.Offset(108))
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	PSHUFL(Imm(0x39), X15, X15)
+	PSHUFL(Imm(0x39), X11, X11)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X6, X6)
+	XORL(Mem{Base: SI}.Offset(160), EDX)
+	XORL(Mem{Base: SI}.Offset(164), ECX)
+	XORL(Mem{Base: SI}.Offset(168), R8L)
+	XORL(Mem{Base: SI}.Offset(172), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(160))
+	MOVL(ECX, Mem{Base: DI}.Offset(164))
+	MOVL(R8L, Mem{Base: DI}.Offset(168))
+	MOVL(R9L, Mem{Base: DI}.Offset(172))
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	XORL(Mem{Base: SI}.Offset(224), EDX)
+	XORL(Mem{Base: SI}.Offset(228), ECX)
+	XORL(Mem{Base: SI}.Offset(232), R8L)
+	XORL(Mem{Base: SI}.Offset(236), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(224))
+	MOVL(ECX, Mem{Base: DI}.Offset(228))
+	MOVL(R8L, Mem{Base: DI}.Offset(232))
+	MOVL(R9L, Mem{Base: DI}.Offset(236))
+	PADDL(Mem{Base: R12}.Offset(160), X13)
+	PADDL(Mem{Base: R12}.Offset(208), X9)
+	PADDL(Mem{Base: R12}.Offset(256), X3)
+	PADDL(Mem{Base: R12}.Offset(96), X2)
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	PSHUFL(Imm(0x39), X13, X13)
+	PSHUFL(Imm(0x39), X9, X9)
+	PSHUFL(Imm(0x39), X3, X3)
+	PSHUFL(Imm(0x39), X2, X2)
+	XORL(Mem{Base: SI}.Offset(48), EDX)
+	XORL(Mem{Base: SI}.Offset(52), ECX)
+	XORL(Mem{Base: SI}.Offset(56), R8L)
+	XORL(Mem{Base: SI}.Offset(60), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(48))
+	MOVL(ECX, Mem{Base: DI}.Offset(52))
+	MOVL(R8L, Mem{Base: DI}.Offset(56))
+	MOVL(R9L, Mem{Base: DI}.Offset(60))
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	PSHUFL(Imm(0x39), X13, X13)
+	PSHUFL(Imm(0x39), X9, X9)
+	PSHUFL(Imm(0x39), X3, X3)
+	PSHUFL(Imm(0x39), X2, X2)
+	XORL(Mem{Base: SI}.Offset(112), EDX)
+	XORL(Mem{Base: SI}.Offset(116), ECX)
+	XORL(Mem{Base: SI}.Offset(120), R8L)
+	XORL(Mem{Base: SI}.Offset(124), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(112))
+	MOVL(ECX, Mem{Base: DI}.Offset(116))
+	MOVL(R8L, Mem{Base: DI}.Offset(120))
+	MOVL(R9L, Mem{Base: DI}.Offset(124))
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	PSHUFL(Imm(0x39), X13, X13)
+	PSHUFL(Imm(0x39), X9, X9)
+	PSHUFL(Imm(0x39), X3, X3)
+	PSHUFL(Imm(0x39), X2, X2)
+	XORL(Mem{Base: SI}.Offset(176), EDX)
+	XORL(Mem{Base: SI}.Offset(180), ECX)
+	XORL(Mem{Base: SI}.Offset(184), R8L)
+	XORL(Mem{Base: SI}.Offset(188), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(176))
+	MOVL(ECX, Mem{Base: DI}.Offset(180))
+	MOVL(R8L, Mem{Base: DI}.Offset(184))
+	MOVL(R9L, Mem{Base: DI}.Offset(188))
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	XORL(Mem{Base: SI}.Offset(240), EDX)
+	XORL(Mem{Base: SI}.Offset(244), ECX)
+	XORL(Mem{Base: SI}.Offset(248), R8L)
+	XORL(Mem{Base: SI}.Offset(252), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(240))
+	MOVL(ECX, Mem{Base: DI}.Offset(244))
+	MOVL(R8L, Mem{Base: DI}.Offset(248))
+	MOVL(R9L, Mem{Base: DI}.Offset(252))
+	MOVQ(Mem{Base: R12}.Offset(352), R9)
+	SUBQ(U32(256), R9)
+	ADDQ(U32(256), RSI)
+	ADDQ(U32(256), RDI)
+	CMPQ(R9, U32(256))
+	JAE(LabelRef("BYTESATLEAST256"))
+	CMPQ(R9, Imm(0))
+	JBE(LabelRef("DONE"))
+}
+
+func BYTESBETWEEN1AND255() {
+	Label("BYTESBETWEEN1AND255")
+	CMPQ(R9, Imm(64))
+	JAE(LabelRef("NOCOPY"))
+	MOVQ(RDI, RDX)
+	LEAQ(Mem{Base: R12}.Offset(360), RDI)
+	MOVQ(R9, RCX)
+	// Hack to get Avo to emit:
+	// 	REP; MOVSB
+	Instruction(&ir.Instruction{Opcode: "REP; MOVSB"})
+	LEAQ(Mem{Base: R12}.Offset(360), RDI)
+	LEAQ(Mem{Base: R12}.Offset(360), RSI)
+}
+
+func NOCOPY() {
+	Label("NOCOPY")
+	MOVQ(R9, Mem{Base: R12}.Offset(352))
+	MOVOA(Mem{Base: R12}.Offset(48), X0)
+	MOVOA(Mem{Base: R12}.Offset(0), X1)
+	MOVOA(Mem{Base: R12}.Offset(16), X2)
+	MOVOA(Mem{Base: R12}.Offset(32), X3)
+	MOVOA(X1, X4)
+	MOVQ(U32(20), RCX)
+}
+
+func MAINLOOP2() {
+	Label("MAINLOOP2")
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X3)
+	PXOR(X6, X3)
+	PADDL(X3, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X3, X3)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X1)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PXOR(X6, X0)
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X1)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X1, X1)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X3)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X3)
+	PADDL(X3, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X3, X3)
+	PXOR(X6, X0)
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X3)
+	PXOR(X6, X3)
+	PADDL(X3, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X3, X3)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X1)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PXOR(X6, X0)
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X1)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X1, X1)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X3)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X3)
+	SUBQ(Imm(4), RCX)
+	PADDL(X3, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PXOR(X7, X7)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X3, X3)
+	PXOR(X6, X0)
+	JA(LabelRef("MAINLOOP2"))
+	PADDL(Mem{Base: R12}.Offset(48), X0)
+	PADDL(Mem{Base: R12}.Offset(0), X1)
+	PADDL(Mem{Base: R12}.Offset(16), X2)
+	PADDL(Mem{Base: R12}.Offset(32), X3)
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X2, X2)
+	PSHUFL(Imm(0x39), X3, X3)
+	XORL(Mem{Base: SI}.Offset(0), ECX)
+	XORL(Mem{Base: SI}.Offset(48), R8L)
+	XORL(Mem{Base: SI}.Offset(32), R9L)
+	XORL(Mem{Base: SI}.Offset(16), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(0))
+	MOVL(R8L, Mem{Base: DI}.Offset(48))
+	MOVL(R9L, Mem{Base: DI}.Offset(32))
+	MOVL(EAX, Mem{Base: DI}.Offset(16))
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X2, X2)
+	PSHUFL(Imm(0x39), X3, X3)
+	XORL(Mem{Base: SI}.Offset(20), ECX)
+	XORL(Mem{Base: SI}.Offset(4), R8L)
+	XORL(Mem{Base: SI}.Offset(52), R9L)
+	XORL(Mem{Base: SI}.Offset(36), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(20))
+	MOVL(R8L, Mem{Base: DI}.Offset(4))
+	MOVL(R9L, Mem{Base: DI}.Offset(52))
+	MOVL(EAX, Mem{Base: DI}.Offset(36))
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X2, X2)
+	PSHUFL(Imm(0x39), X3, X3)
+	XORL(Mem{Base: SI}.Offset(40), ECX)
+	XORL(Mem{Base: SI}.Offset(24), R8L)
+	XORL(Mem{Base: SI}.Offset(8), R9L)
+	XORL(Mem{Base: SI}.Offset(56), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(40))
+	MOVL(R8L, Mem{Base: DI}.Offset(24))
+	MOVL(R9L, Mem{Base: DI}.Offset(8))
+	MOVL(EAX, Mem{Base: DI}.Offset(56))
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	XORL(Mem{Base: SI}.Offset(60), ECX)
+	XORL(Mem{Base: SI}.Offset(44), R8L)
+	XORL(Mem{Base: SI}.Offset(28), R9L)
+	XORL(Mem{Base: SI}.Offset(12), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(60))
+	MOVL(R8L, Mem{Base: DI}.Offset(44))
+	MOVL(R9L, Mem{Base: DI}.Offset(28))
+	MOVL(EAX, Mem{Base: DI}.Offset(12))
+	MOVQ(Mem{Base: R12}.Offset(352), R9)
+	MOVL(Mem{Base: R12}.Offset(16), ECX)
+	MOVL(Mem{Base: R12}.Offset(36), R8L)
+	ADDQ(Imm(1), RCX)
+	SHLQ(Imm(32), R8)
+	ADDQ(R8, RCX)
+	MOVQ(RCX, R8)
+	SHRQ(Imm(32), R8)
+	MOVL(ECX, Mem{Base: R12}.Offset(16))
+	MOVL(R8L, Mem{Base: R12}.Offset(36))
+	CMPQ(R9, Imm(64))
+	JA(LabelRef("BYTESATLEAST65"))
+	JAE(LabelRef("BYTESATLEAST64"))
+	MOVQ(RDI, RSI)
+	MOVQ(RDX, RDI)
+	MOVQ(R9, RCX)
+	// Hack to get Avo to emit:
+	// 	REP; MOVSB
+	Instruction(&ir.Instruction{Opcode: "REP; MOVSB"})
+}

sha3/shake.go

@@ -85,9 +85,9 @@ func newCShake(N, S []byte, rate, outputLen int, dsbyte byte) ShakeHash {
 
 	// leftEncode returns max 9 bytes
 	c.initBlock = make([]byte, 0, 9*2+len(N)+len(S))
-	c.initBlock = append(c.initBlock, leftEncode(uint64(len(N)*8))...)
+	c.initBlock = append(c.initBlock, leftEncode(uint64(len(N))*8)...)
 	c.initBlock = append(c.initBlock, N...)
-	c.initBlock = append(c.initBlock, leftEncode(uint64(len(S)*8))...)
+	c.initBlock = append(c.initBlock, leftEncode(uint64(len(S))*8)...)
 	c.initBlock = append(c.initBlock, S...)
 	c.Write(bytepad(c.initBlock, c.rate))
 	return &c

ssh/agent/keyring.go

@@ -175,6 +175,15 @@ func (r *keyring) Add(key AddedKey) error {
 		p.expire = &t
 	}
 
+	// If we already have a Signer with the same public key, replace it with the
+	// new one.
+	for idx, k := range r.keys {
+		if bytes.Equal(k.signer.PublicKey().Marshal(), p.signer.PublicKey().Marshal()) {
+			r.keys[idx] = p
+			return nil
+		}
+	}
+
 	r.keys = append(r.keys, p)
 
 	return nil

ssh/agent/keyring_test.go

@@ -29,6 +29,10 @@ func validateListedKeys(t *testing.T, a Agent, expectedKeys []string) {
 		t.Fatalf("failed to list keys: %v", err)
 		return
 	}
+	if len(listedKeys) != len(expectedKeys) {
+		t.Fatalf("expeted %d key, got %d", len(expectedKeys), len(listedKeys))
+		return
+	}
 	actualKeys := make(map[string]bool)
 	for _, key := range listedKeys {
 		actualKeys[key.Comment] = true
@@ -74,3 +78,45 @@ func TestKeyringAddingAndRemoving(t *testing.T) {
 	}
 	validateListedKeys(t, k, []string{})
 }
+
+func TestAddDuplicateKey(t *testing.T) {
+	keyNames := []string{"rsa", "user"}
+
+	k := NewKeyring()
+	for _, keyName := range keyNames {
+		addTestKey(t, k, keyName)
+	}
+	validateListedKeys(t, k, keyNames)
+	// Add the keys again.
+	for _, keyName := range keyNames {
+		addTestKey(t, k, keyName)
+	}
+	validateListedKeys(t, k, keyNames)
+	// Add an existing key with an updated comment.
+	keyName := keyNames[0]
+	addedKey := AddedKey{
+		PrivateKey: testPrivateKeys[keyName],
+		Comment:    "comment updated",
+	}
+	err := k.Add(addedKey)
+	if err != nil {
+		t.Fatalf("failed to add key %q: %v", keyName, err)
+	}
+	// Check the that key is found and the comment was updated.
+	keys, err := k.List()
+	if err != nil {
+		t.Fatalf("failed to list keys: %v", err)
+	}
+	if len(keys) != len(keyNames) {
+		t.Fatalf("expected %d keys, got %d", len(keyNames), len(keys))
+	}
+	isFound := false
+	for _, key := range keys {
+		if key.Comment == addedKey.Comment {
+			isFound = true
+		}
+	}
+	if !isFound {
+		t.Fatal("key with the updated comment not found")
+	}
+}

ssh/client_auth_test.go

@@ -641,17 +641,28 @@ func TestClientAuthMaxAuthTries(t *testing.T) {
 		defer c1.Close()
 		defer c2.Close()
 
-		go newServer(c1, serverConfig)
-		_, _, _, err = NewClientConn(c2, "", clientConfig)
-		if tries > 2 {
-			if err == nil {
+		errCh := make(chan error, 1)
+
+		go func() {
+			_, err := newServer(c1, serverConfig)
+			errCh <- err
+		}()
+		_, _, _, cliErr := NewClientConn(c2, "", clientConfig)
+		srvErr := <-errCh
+
+		if tries > serverConfig.MaxAuthTries {
+			if cliErr == nil {
 				t.Fatalf("client: got no error, want %s", expectedErr)
-			} else if err.Error() != expectedErr.Error() {
+			} else if cliErr.Error() != expectedErr.Error() {
 				t.Fatalf("client: got %s, want %s", err, expectedErr)
 			}
+			var authErr *ServerAuthError
+			if !errors.As(srvErr, &authErr) {
+				t.Errorf("expected ServerAuthError, got: %v", srvErr)
+			}
 		} else {
-			if err != nil {
-				t.Fatalf("client: got %s, want no error", err)
+			if cliErr != nil {
+				t.Fatalf("client: got %s, want no error", cliErr)
 			}
 		}
 	}

ssh/server.go

@@ -510,8 +510,8 @@ userAuthLoop:
 			if err := s.transport.writePacket(Marshal(discMsg)); err != nil {
 				return nil, err
 			}
-
-			return nil, discMsg
+			authErrs = append(authErrs, discMsg)
+			return nil, &ServerAuthError{Errors: authErrs}
 		}
 
 		var userAuthReq userAuthRequestMsg

x509roots/fallback/bundle.go

@@ -2794,6 +2794,79 @@ I50mD1hp/Ed+stCNi5O/KU9DaXR2Z0vPB4zmAve14bRDtUstFJ/53CYNv6ZHdAbY
 iNE6KTCEztI5gGIbqMdXSbxqVVFnFUq+NQfk1XWYN3kwFNspnWzFacxHVaIw98xc
 f8LDmBxrThaA63p4ZUWiABqvDA1VZDRIuJK58bRQKfJPIx/abKwfROHdI3hRW8cW
 -----END CERTIFICATE-----
+# CN=SecureSign Root CA12,O=Cybertrust Japan Co.\, Ltd.,C=JP
+# 3f034bb5704d44b2d08545a02057de93ebf3905fce721acbc730c06ddaee904e
+-----BEGIN CERTIFICATE-----
+MIIDcjCCAlqgAwIBAgIUZvnHwa/swlG07VOX5uaCwysckBYwDQYJKoZIhvcNAQEL
+BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u
+LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExMjAeFw0yMDA0MDgw
+NTM2NDZaFw00MDA0MDgwNTM2NDZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD
+eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS
+b290IENBMTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6OcE3emhF
+KxS06+QT61d1I02PJC0W6K6OyX2kVzsqdiUzg2zqMoqUm048luT9Ub+ZyZN+v/mt
+p7JIKwccJ/VMvHASd6SFVLX9kHrko+RRWAPNEHl57muTH2SOa2SroxPjcf59q5zd
+J1M3s6oYwlkm7Fsf0uZlfO+TvdhYXAvA42VvPMfKWeP+bl+sg779XSVOKik71gur
+FzJ4pOE+lEa+Ym6b3kaosRbnhW70CEBFEaCeVESE99g2zvVQR9wsMJvuwPWW0v4J
+hscGWa5Pro4RmHvzC1KqYiaqId+OJTN5lxZJjfU+1UefNzFJM3IFTQy2VYzxV4+K
+h9GtxRESOaCtAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD
+AgEGMB0GA1UdDgQWBBRXNPN0zwRL1SXm8UC2LEzZLemgrTANBgkqhkiG9w0BAQsF
+AAOCAQEAPrvbFxbS8hQBICw4g0utvsqFepq2m2um4fylOqyttCg6r9cBg0krY6Ld
+mmQOmFxv3Y67ilQiLUoT865AQ9tPkbeGGuwAtEGBpE/6aouIs3YIcipJQMPTw4WJ
+mBClnW8Zt7vPemVV2zfrPIpyMpcemik+rY3moxtt9XUa5rBouVui7mlHJzWhhpmA
+8zNL4WukJsPvdFlseqJkth5Ew1DgDzk9qTPxpfPSvWKErI4cqc1avTc7bgoitPQV
+55FYxTpE05Uo2cBl6XLK0A+9H7MV2anjpEcJnuDLN/v9vZfVvhgaaaI5gdka9at/
+yOPiZwud9AzqVN/Ssq+xIvEg37xEHA==
+-----END CERTIFICATE-----
+# CN=SecureSign Root CA14,O=Cybertrust Japan Co.\, Ltd.,C=JP
+# 4b009c1034494f9ab56bba3ba1d62731fc4d20d8955adcec10a925607261e338
+-----BEGIN CERTIFICATE-----
+MIIFcjCCA1qgAwIBAgIUZNtaDCBO6Ncpd8hQJ6JaJ90t8sswDQYJKoZIhvcNAQEM
+BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u
+LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNDAeFw0yMDA0MDgw
+NzA2MTlaFw00NTA0MDgwNzA2MTlaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD
+eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS
+b290IENBMTQwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDF0nqh1oq/
+FjHQmNE6lPxauG4iwWL3pwon71D2LrGeaBLwbCRjOfHw3xDG3rdSINVSW0KZnvOg
+vlIfX8xnbacuUKLBl422+JX1sLrcneC+y9/3OPJH9aaakpUqYllQC6KxNedlsmGy
+6pJxaeQp8E+BgQQ8sqVb1MWoWWd7VRxJq3qdwudzTe/NCcLEVxLbAQ4jeQkHO6Lo
+/IrPj8BGJJw4J+CDnRugv3gVEOuGTgpa/d/aLIJ+7sr2KeH6caH3iGicnPCNvg9J
+kdjqOvn90Ghx2+m1K06Ckm9mH+Dw3EzsytHqunQG+bOEkJTRX45zGRBdAuVwpcAQ
+0BB8b8VYSbSwbprafZX1zNoCr7gsfXmPvkPx+SgojQlD+Ajda8iLLCSxjVIHvXib
+y8posqTdDEx5YMaZ0ZPxMBoH064iwurO8YQJzOAUbn8/ftKChazcqRZOhaBgy/ac
+18izju3Gm5h1DVXoX+WViwKkrkMpKBGk5hIwAUt1ax5mnXkvpXYvHUC0bcl9eQjs
+0Wq2XSqypWa9a4X0dFbD9ed1Uigspf9mR6XU/v6eVL9lfgHWMI+lNpyiUBzuOIAB
+SMbHdPTGrMNASRZhdCyvjG817XsYAFs2PJxQDcqSMxDxJklt33UkN4Ii1+iW/RVL
+ApY+B3KVfqs9TC7XyvDf4Fg/LS8EmjijAQIDAQABo0IwQDAPBgNVHRMBAf8EBTAD
+AQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUBpOjCl4oaTeqYR3r6/wtbyPk
+86AwDQYJKoZIhvcNAQEMBQADggIBAJaAcgkGfpzMkwQWu6A6jZJOtxEaCnFxEM0E
+rX+lRVAQZk5KQaID2RFPeje5S+LGjzJmdSX7684/AykmjbgWHfYfM25I5uj4V7Ib
+ed87hwriZLoAymzvftAj63iP/2SbNDefNWWipAA9EiOWWF3KY4fGoweITedpdopT
+zfFP7ELyk+OZpDc8h7hi2/DsHzc/N19DzFGdtfCXwreFamgLRB7lUe6TzktuhsHS
+DCRZNhqfLJGP4xjblJUK7ZGqDpncllPjYYPGFrojutzdfhrGe0K22VoF3Jpf1d+4
+2kd92jjbrDnVHmtsKheMYc2xbXIBw8MgAGJoFjHVdqqGuw6qnsb58Nn4DSEC5MUo
+FlkRudlpcyqSeLiSV5sI8jrlL5WwWLdrIBRtFO8KvH7YVdiI2i/6GaX7i+B/OfVy
+K4XELKzvGUWSTLNhB9xNH27SgRNcmvMSZ4PPmz+Ln52kuaiWA3rF7iDeM9ovnhp6
+dB7h7sxaOgTdsxoEqBRjrLdHEoOabPXm6RUVkRqEGQ6UROcSjiVbgGcZ3GOTEAtl
+Lor6CZpO2oYofaphNdgOpygau1LgePhsumywbrmHXumZNTfxPWQrqaA0k89jL9WB
+365jJ6UeTo3cKXhZ+PmhIIynJkBugnLNeLLIjzwec+fBH7/PzqUqm9tEZDKgu39c
+JRNItX+S
+-----END CERTIFICATE-----
+# CN=SecureSign Root CA15,O=Cybertrust Japan Co.\, Ltd.,C=JP
+# e778f0f095fe843729cd1a0082179e5314a9c291442805e1fb1d8fb6b8886c3a
+-----BEGIN CERTIFICATE-----
+MIICIzCCAamgAwIBAgIUFhXHw9hJp75pDIqI7fBw+d23PocwCgYIKoZIzj0EAwMw
+UTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28uLCBM
+dGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNTAeFw0yMDA0MDgwODMy
+NTZaFw00NTA0MDgwODMyNTZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpDeWJl
+cnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBSb290
+IENBMTUwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQLUHSNZDKZmbPSYAi4Io5GdCx4
+wCtELW1fHcmuS1Iggz24FG1Th2CeX2yF2wYUleDHKP+dX+Sq8bOLbe1PL0vJSpSR
+ZHX+AezB2Ot6lHhWGENfa4HL9rzatAy2KZMIaY+jQjBAMA8GA1UdEwEB/wQFMAMB
+Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBTrQciu/NWeUUj1vYv0hyCTQSvT
+9DAKBggqhkjOPQQDAwNoADBlAjEA2S6Jfl5OpBEHvVnCB96rMjhTKkZEBhd6zlHp
+4P9mLQlO4E/0BdGF9jVg3PVys0Z9AjBEmEYagoUeYWmJSwdLZrWeqrqgHkHZAXQ6
+bkU6iYAZezKYVWOr62Nuk22rGwlgMU4=
+-----END CERTIFICATE-----
 # CN=SecureSign RootCA11,O=Japan Certification Services\, Inc.,C=JP
 # bf0feefb9e3a581ad5f9e9db7589985743d261085c4d314f6f5d7259aa421612
 -----BEGIN CERTIFICATE-----
@@ -3062,6 +3135,40 @@ WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ
 e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p
 TpPDpFQUWw==
 -----END CERTIFICATE-----
+# CN=TWCA CYBER Root CA,OU=Root CA,O=TAIWAN-CA,C=TW
+# 3f63bb2814be174ec8b6439cf08d6d56f0b7c405883a5648a334424d6b3ec558
+-----BEGIN CERTIFICATE-----
+MIIFjTCCA3WgAwIBAgIQQAE0jMIAAAAAAAAAATzyxjANBgkqhkiG9w0BAQwFADBQ
+MQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FOLUNBMRAwDgYDVQQLEwdSb290
+IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3QgQ0EwHhcNMjIxMTIyMDY1NDI5
+WhcNNDcxMTIyMTU1OTU5WjBQMQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FO
+LUNBMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3Qg
+Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDG+Moe2Qkgfh1sTs6P
+40czRJzHyWmqOlt47nDSkvgEs1JSHWdyKKHfi12VCv7qze33Kc7wb3+szT3vsxxF
+avcokPFhV8UMxKNQXd7UtcsZyoC5dc4pztKFIuwCY8xEMCDa6pFbVuYdHNWdZsc/
+34bKS1PE2Y2yHer43CdTo0fhYcx9tbD47nORxc5zb87uEB8aBs/pJ2DFTxnk684i
+JkXXYJndzk834H/nY62wuFm40AZoNWDTNq5xQwTxaWV4fPMf88oon1oglWa0zbfu
+j3ikRRjpJi+NmykosaS3Om251Bw4ckVYsV7r8Cibt4LK/c/WMw+f+5eesRycnupf
+Xtuq3VTpMCEobY5583WSjCb+3MX2w7DfRFlDo7YDKPYIMKoNM+HvnKkHIuNZW0CP
+2oi3aQiotyMuRAlZN1vH4xfyIutuOVLF3lSnmMlLIJXcRolftBL5hSmO68gnFSDA
+S9TMfAxsNAwmmyYxpjyn9tnQS6Jk/zuZQXLB4HCX8SS7K8R0IrGsayIyJNN4KsDA
+oS/xUgXJP+92ZuJF2A09rZXIx4kmyA+upwMu+8Ff+iDhcK2wZSA3M2Cw1a/XDBzC
+kHDXShi8fgGwsOsVHkQGzaRP6AzRwyAQ4VRlnrZR0Bp2a0JaWHY06rc3Ga4udfmW
+5cFZ95RXKSWNOkyrTZpB0F8mAwIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYD
+VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBSdhWEUfMFib5do5E83QOGt4A1WNzAd
+BgNVHQ4EFgQUnYVhFHzBYm+XaORPN0DhreANVjcwDQYJKoZIhvcNAQEMBQADggIB
+AGSPesRiDrWIzLjHhg6hShbNcAu3p4ULs3a2D6f/CIsLJc+o1IN1KriWiLb73y0t
+tGlTITVX1olNc79pj3CjYcya2x6a4CD4bLubIp1dhDGaLIrdaqHXKGnK/nZVekZn
+68xDiBaiA9a5F/gZbG0jAn/xX9AKKSM70aoK7akXJlQKTcKlTfjF/biBzysseKNn
+TKkHmvPfXvt89YnNdJdhEGoHK4Fa0o635yDRIG4kqIQnoVesqlVYL9zZyvpoBJ7t
+RCT5dEA7IzOrg1oYJkK2bVS1FmAwbLGg+LhBoF1JSdJlBTrq/p1hvIbZv97Tujqx
+f36SNI7JAG7cmL3c7IAFrQI932XtCwP39xaEBDG6k5TY8hL4iuO/Qq+n1M0RFxbI
+Qh0UqEL20kCGoE8jypZFVmAGzbdVAaYBlGX+bgUJurSkquLvWL69J1bY73NxW0Qz
+8ppy6rBePm6pUlvscG21h483XjyMnM7k8M4MZ0HMzvaAq07MTFb1wWFZk7Q+ptq4
+NxKfKjLji7gh7MMrZQzvIt6IKTtM1/r+t+FHvpw+PoP7UV31aPcuIYXcv/Fa4nzX
+xeSDwWrruoBa3lwtcHb4yOWHh8qgnaHlIhInD0Q9HWzq1MKLL295q39QpsQZp6F6
+t5b5wR9iWqJDB0BeJsas7a5wFsWqynKKTbDPAYsDP27X
+-----END CERTIFICATE-----
 # CN=TWCA Global Root CA,OU=Root CA,O=TAIWAN-CA,C=TW
 # 59769007f7685d0fcd50872f9f95d5755a5b2b457d81f3692b610a98672f0e1b
 -----BEGIN CERTIFICATE-----