all: fix printf(var) mistakes detected by latest printf checker

These were problematic but previously easy to miss. They're now
easy to spot thanks to build failures at Go tip as of CL 610736.

For golang/go#68796.

Change-Id: I167f2cce2376b4070460389c673d973e4521d3dc
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/610797
Auto-Submit: Dmitri Shuralyov <dmitshur@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Alan Donovan <adonovan@google.com>

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
+Copyright 2009 The Go Authors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
 copyright notice, this list of conditions and the following disclaimer
 in the documentation and/or other materials provided with the
 distribution.
-   * Neither the name of Google Inc. nor the names of its
+   * Neither the name of Google LLC nor the names of its
 contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 

acme/autocert/internal/acmetest/ca.go

@@ -308,7 +308,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		}
 
 		if err := decodePayload(&req, r.Body); err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 
@@ -328,7 +328,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 			Identifiers []struct{ Value string }
 		}
 		if err := decodePayload(&req, r.Body); err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		ca.mu.Lock()
@@ -352,7 +352,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		defer ca.mu.Unlock()
 		o, err := ca.storedOrder(strings.TrimPrefix(r.URL.Path, "/orders/"))
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		if err := json.NewEncoder(w).Encode(o); err != nil {
@@ -412,7 +412,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		orderID := strings.TrimPrefix(r.URL.Path, "/new-cert/")
 		o, err := ca.storedOrder(orderID)
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		if o.Status != acme.StatusReady {
@@ -427,7 +427,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		b, _ := base64.RawURLEncoding.DecodeString(req.CSR)
 		csr, err := x509.ParseCertificateRequest(b)
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		// Issue the certificate.
@@ -449,7 +449,7 @@ func (ca *CAServer) handle(w http.ResponseWriter, r *http.Request) {
 		defer ca.mu.Unlock()
 		o, err := ca.storedOrder(strings.TrimPrefix(r.URL.Path, "/issued-cert/"))
 		if err != nil {
-			ca.httpErrorf(w, http.StatusBadRequest, err.Error())
+			ca.httpErrorf(w, http.StatusBadRequest, "%v", err)
 			return
 		}
 		if o.Status != acme.StatusValid {

acme/http.go

@@ -15,6 +15,7 @@ import (
 	"io"
 	"math/big"
 	"net/http"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"time"
@@ -271,9 +272,27 @@ func (c *Client) httpClient() *http.Client {
 }
 
 // packageVersion is the version of the module that contains this package, for
-// sending as part of the User-Agent header. It's set in version_go112.go.
+// sending as part of the User-Agent header.
 var packageVersion string
 
+func init() {
+	// Set packageVersion if the binary was built in modules mode and x/crypto
+	// was not replaced with a different module.
+	info, ok := debug.ReadBuildInfo()
+	if !ok {
+		return
+	}
+	for _, m := range info.Deps {
+		if m.Path != "golang.org/x/crypto" {
+			continue
+		}
+		if m.Replace == nil {
+			packageVersion = m.Version
+		}
+		break
+	}
+}
+
 // userAgent returns the User-Agent header value. It includes the package name,
 // the module version (if available), and the c.UserAgent value (if set).
 func (c *Client) userAgent() string {

acme/version_go112.go

@@ -1,27 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build go1.12
-
-package acme
-
-import "runtime/debug"
-
-func init() {
-	// Set packageVersion if the binary was built in modules mode and x/crypto
-	// was not replaced with a different module.
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return
-	}
-	for _, m := range info.Deps {
-		if m.Path != "golang.org/x/crypto" {
-			continue
-		}
-		if m.Replace == nil {
-			packageVersion = m.Version
-		}
-		break
-	}
-}

argon2/_asm/blamka_amd64.go

@@ -0,0 +1,287 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/argon2"
+)
+
+//go:generate go run . -out ../blamka_amd64.s -pkg argon2
+
+func main() {
+	Package("golang.org/x/crypto/argon2")
+	ConstraintExpr("amd64,gc,!purego")
+
+	blamkaSSE4()
+	mixBlocksSSE2()
+	xorBlocksSSE2()
+	Generate()
+}
+
+func blamkaSSE4() {
+	Implement("blamkaSSE4")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("b"), RAX)
+
+	c40 := c40_DATA()
+	c48 := c48_DATA()
+
+	MOVOU(c40, X10)
+	MOVOU(c48, X11)
+
+	BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11)
+
+	BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11)
+	RET()
+}
+
+func mixBlocksSSE2() {
+	Implement("mixBlocksSSE2")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("out"), RDX)
+	Load(Param("a"), RAX)
+	Load(Param("b"), RBX)
+	Load(Param("c"), RCX)
+	MOVQ(U32(128), RDI)
+
+	Label("loop")
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: BX}.Offset(0), X1)
+	MOVOU(Mem{Base: CX}.Offset(0), X2)
+	PXOR(X1, X0)
+	PXOR(X2, X0)
+	MOVOU(X0, Mem{Base: DX}.Offset(0))
+	ADDQ(Imm(16), RAX)
+	ADDQ(Imm(16), RBX)
+	ADDQ(Imm(16), RCX)
+	ADDQ(Imm(16), RDX)
+	SUBQ(Imm(2), RDI)
+	JA(LabelRef("loop"))
+	RET()
+}
+
+func xorBlocksSSE2() {
+	Implement("xorBlocksSSE2")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("out"), RDX)
+	Load(Param("a"), RAX)
+	Load(Param("b"), RBX)
+	Load(Param("c"), RCX)
+	MOVQ(U32(128), RDI)
+
+	Label("loop")
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: BX}.Offset(0), X1)
+	MOVOU(Mem{Base: CX}.Offset(0), X2)
+	MOVOU(Mem{Base: DX}.Offset(0), X3)
+	PXOR(X1, X0)
+	PXOR(X2, X0)
+	PXOR(X3, X0)
+	MOVOU(X0, Mem{Base: DX}.Offset(0))
+	ADDQ(Imm(16), RAX)
+	ADDQ(Imm(16), RBX)
+	ADDQ(Imm(16), RCX)
+	ADDQ(Imm(16), RDX)
+	SUBQ(Imm(2), RDI)
+	JA(LabelRef("loop"))
+	RET()
+}
+
+func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v6, t1)
+	PUNPCKLQDQ(v6, t2)
+	PUNPCKHQDQ(v7, v6)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(v7, t2)
+	MOVO(t1, v7)
+	MOVO(v2, t1)
+	PUNPCKHQDQ(t2, v7)
+	PUNPCKLQDQ(v3, t2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v3)
+}
+
+func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v2, t1)
+	PUNPCKLQDQ(v2, t2)
+	PUNPCKHQDQ(v3, v2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(v3, t2)
+	MOVO(t1, v3)
+	MOVO(v6, t1)
+	PUNPCKHQDQ(t2, v3)
+	PUNPCKLQDQ(v7, t2)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v7)
+}
+
+func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48 VecPhysical) {
+	MOVO(v0, t0)
+	PMULULQ(v2, t0)
+	PADDQ(v2, v0)
+	PADDQ(t0, v0)
+	PADDQ(t0, v0)
+	PXOR(v0, v6)
+	PSHUFD(Imm(0xB1), v6, v6)
+	MOVO(v4, t0)
+	PMULULQ(v6, t0)
+	PADDQ(v6, v4)
+	PADDQ(t0, v4)
+	PADDQ(t0, v4)
+	PXOR(v4, v2)
+	PSHUFB(c40, v2)
+	MOVO(v0, t0)
+	PMULULQ(v2, t0)
+	PADDQ(v2, v0)
+	PADDQ(t0, v0)
+	PADDQ(t0, v0)
+	PXOR(v0, v6)
+	PSHUFB(c48, v6)
+	MOVO(v4, t0)
+	PMULULQ(v6, t0)
+	PADDQ(v6, v4)
+	PADDQ(t0, v4)
+	PADDQ(t0, v4)
+	PXOR(v4, v2)
+	MOVO(v2, t0)
+	PADDQ(v2, t0)
+	PSRLQ(Imm(63), v2)
+	PXOR(t0, v2)
+	MOVO(v1, t0)
+	PMULULQ(v3, t0)
+	PADDQ(v3, v1)
+	PADDQ(t0, v1)
+	PADDQ(t0, v1)
+	PXOR(v1, v7)
+	PSHUFD(Imm(0xB1), v7, v7)
+	MOVO(v5, t0)
+	PMULULQ(v7, t0)
+	PADDQ(v7, v5)
+	PADDQ(t0, v5)
+	PADDQ(t0, v5)
+	PXOR(v5, v3)
+	PSHUFB(c40, v3)
+	MOVO(v1, t0)
+	PMULULQ(v3, t0)
+	PADDQ(v3, v1)
+	PADDQ(t0, v1)
+	PADDQ(t0, v1)
+	PXOR(v1, v7)
+	PSHUFB(c48, v7)
+	MOVO(v5, t0)
+	PMULULQ(v7, t0)
+	PADDQ(v7, v5)
+	PADDQ(t0, v5)
+	PADDQ(t0, v5)
+	PXOR(v5, v3)
+	MOVO(v3, t0)
+	PADDQ(v3, t0)
+	PSRLQ(Imm(63), v3)
+	PXOR(t0, v3)
+}
+
+func LOAD_MSG_0(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(Mem{Base: block}.Offset(8*(off+(i*2))), r)
+	}
+}
+
+func STORE_MSG_0(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(r, Mem{Base: block}.Offset(8*(off+(i*2))))
+	}
+}
+
+func LOAD_MSG_1(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(Mem{Base: block}.Offset(8*off+i*16*8), r)
+	}
+}
+
+func STORE_MSG_1(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(r, Mem{Base: block}.Offset(8*off+i*16*8))
+	}
+}
+
+func BLAMKA_ROUND_0(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) {
+	LOAD_MSG_0(block, off)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1)
+	STORE_MSG_0(block, off)
+}
+
+func BLAMKA_ROUND_1(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) {
+	LOAD_MSG_1(block, off)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1)
+	STORE_MSG_1(block, off)
+}
+
+// ##------------------DATA SECTION-------------------##
+
+var c40_DATA_ptr, c48_DATA_ptr *Mem
+
+func c40_DATA() Mem {
+	if c40_DATA_ptr != nil {
+		return *c40_DATA_ptr
+	}
+
+	c40_DATA := GLOBL("·c40", NOPTR|RODATA)
+	c40_DATA_ptr = &c40_DATA
+	DATA(0x00, U64(0x0201000706050403))
+	DATA(0x08, U64(0x0a09080f0e0d0c0b))
+	return c40_DATA
+}
+func c48_DATA() Mem {
+	if c48_DATA_ptr != nil {
+		return *c48_DATA_ptr
+	}
+
+	c48_DATA := GLOBL("·c48", NOPTR|RODATA)
+	c48_DATA_ptr = &c48_DATA
+	DATA(0x00, U64(0x0100070605040302))
+	DATA(0x08, U64(0x09080f0e0d0c0b0a))
+	return c48_DATA
+}

argon2/_asm/go.mod

@@ -0,0 +1,15 @@
+module argon2/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

argon2/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

bcrypt/bcrypt.go

@@ -4,7 +4,7 @@
 
 // Package bcrypt implements Provos and Mazières's bcrypt adaptive hashing
 // algorithm. See http://www.usenix.org/event/usenix99/provos/provos.pdf
-package bcrypt // import "golang.org/x/crypto/bcrypt"
+package bcrypt
 
 // The code is a port of Provos and Mazières's C implementation.
 import (

blake2b/_asm/AVX2/go.mod

@@ -0,0 +1,16 @@
+module blake2b/_asm/AVX2
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

blake2b/_asm/AVX2/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

blake2b/_asm/standard/blake2b_amd64_asm.go

@@ -0,0 +1,361 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/blake2b"
+)
+
+//go:generate go run . -out ../../blake2b_amd64.s -pkg blake2b
+
+const ThatPeskyUnicodeDot = "\u00b7"
+
+var iv0_DATA_ptr, iv1_DATA_ptr, iv2_DATA_ptr, iv3_DATA_ptr, c40_DATA_ptr, c48_DATA_ptr *Mem
+
+func main() {
+	Package("golang.org/x/crypto/blake2b")
+	ConstraintExpr("amd64,gc,!purego")
+	hashBlocksSSE4()
+	Generate()
+}
+
+func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v6, t1)
+	PUNPCKLQDQ(v6, t2)
+	PUNPCKHQDQ(v7, v6)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(v7, t2)
+	MOVO(t1, v7)
+	MOVO(v2, t1)
+	PUNPCKHQDQ(t2, v7)
+	PUNPCKLQDQ(v3, t2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v3)
+}
+
+func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v2, t1)
+	PUNPCKLQDQ(v2, t2)
+	PUNPCKHQDQ(v3, v2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(v3, t2)
+	MOVO(t1, v3)
+	MOVO(v6, t1)
+	PUNPCKHQDQ(t2, v3)
+	PUNPCKLQDQ(v7, t2)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v7)
+}
+
+func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7 VecPhysical, m0, m1, m2, m3 Op, t0, c40, c48 VecPhysical) {
+	PADDQ(m0, v0)
+	PADDQ(m1, v1)
+	PADDQ(v2, v0)
+	PADDQ(v3, v1)
+	PXOR(v0, v6)
+	PXOR(v1, v7)
+	PSHUFD(Imm(0xB1), v6, v6)
+	PSHUFD(Imm(0xB1), v7, v7)
+	PADDQ(v6, v4)
+	PADDQ(v7, v5)
+	PXOR(v4, v2)
+	PXOR(v5, v3)
+	PSHUFB(c40, v2)
+	PSHUFB(c40, v3)
+	PADDQ(m2, v0)
+	PADDQ(m3, v1)
+	PADDQ(v2, v0)
+	PADDQ(v3, v1)
+	PXOR(v0, v6)
+	PXOR(v1, v7)
+	PSHUFB(c48, v6)
+	PSHUFB(c48, v7)
+	PADDQ(v6, v4)
+	PADDQ(v7, v5)
+	PXOR(v4, v2)
+	PXOR(v5, v3)
+	MOVOU(v2, t0)
+	PADDQ(v2, t0)
+	PSRLQ(Imm(63), v2)
+	PXOR(t0, v2)
+	MOVOU(v3, t0)
+	PADDQ(v3, t0)
+	PSRLQ(Imm(63), v3)
+	PXOR(t0, v3)
+}
+
+func LOAD_MSG(m0, m1, m2, m3 VecPhysical, src GPPhysical, i0, i1, i2, i3, i4, i5, i6, i7 int) {
+	MOVQ(Mem{Base: src}.Offset(i0*8), m0)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i1*8), m0)
+	MOVQ(Mem{Base: src}.Offset(i2*8), m1)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i3*8), m1)
+	MOVQ(Mem{Base: src}.Offset(i4*8), m2)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i5*8), m2)
+	MOVQ(Mem{Base: src}.Offset(i6*8), m3)
+	PINSRQ(Imm(1), Mem{Base: src}.Offset(i7*8), m3)
+}
+
+func hashBlocksSSE4() {
+	Implement("hashBlocksSSE4")
+	Attributes(4)
+	AllocLocal(288) // frame size = 272 + 16 byte alignment
+
+	Load(Param("h"), RAX)
+	Load(Param("c"), RBX)
+	Load(Param("flag"), RCX)
+	Load(Param("blocks").Base(), RSI)
+	Load(Param("blocks").Len(), RDI)
+
+	MOVQ(RSP, R10)
+	ADDQ(Imm(15), R10)
+	ANDQ(I32(-16), R10)
+
+	iv3 := iv3_DATA()
+	MOVOU(iv3, X0)
+	MOVO(X0, Mem{Base: R10}.Offset(0))
+	XORQ(RCX, Mem{Base: R10}.Offset(0)) // 0(R10) = ·iv3 ^ (CX || 0)
+
+	c40 := c40_DATA()
+	c48 := c48_DATA()
+	MOVOU(c40, X13)
+	MOVOU(c48, X14)
+
+	MOVOU(Mem{Base: AX}.Offset(0), X12)
+	MOVOU(Mem{Base: AX}.Offset(16), X15)
+
+	MOVQ(Mem{Base: BX}.Offset(0), R8)
+	MOVQ(Mem{Base: BX}.Offset(8), R9)
+
+	Label("loop")
+	ADDQ(Imm(128), R8)
+	CMPQ(R8, Imm(128))
+	JGE(LabelRef("noinc"))
+	INCQ(R9)
+
+	Label("noinc")
+	MOVQ(R8, X8)
+	PINSRQ(Imm(1), R9, X8)
+
+	iv0 := iv0_DATA()
+	iv1 := iv1_DATA()
+	iv2 := iv2_DATA()
+
+	MOVO(X12, X0)
+	MOVO(X15, X1)
+	MOVOU(Mem{Base: AX}.Offset(32), X2)
+	MOVOU(Mem{Base: AX}.Offset(48), X3)
+	MOVOU(iv0, X4)
+	MOVOU(iv1, X5)
+	MOVOU(iv2, X6)
+
+	PXOR(X8, X6)
+	MOVO(Mem{Base: R10}.Offset(0), X7)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7)
+	MOVO(X8, Mem{Base: R10}.Offset(16))
+	MOVO(X9, Mem{Base: R10}.Offset(32))
+	MOVO(X10, Mem{Base: R10}.Offset(48))
+	MOVO(X11, Mem{Base: R10}.Offset(64))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 8, 10, 12, 14, 9, 11, 13, 15)
+	MOVO(X8, Mem{Base: R10}.Offset(80))
+	MOVO(X9, Mem{Base: R10}.Offset(96))
+	MOVO(X10, Mem{Base: R10}.Offset(112))
+	MOVO(X11, Mem{Base: R10}.Offset(128))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6)
+	MOVO(X8, Mem{Base: R10}.Offset(144))
+	MOVO(X9, Mem{Base: R10}.Offset(160))
+	MOVO(X10, Mem{Base: R10}.Offset(176))
+	MOVO(X11, Mem{Base: R10}.Offset(192))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 1, 0, 11, 5, 12, 2, 7, 3)
+	MOVO(X8, Mem{Base: R10}.Offset(208))
+	MOVO(X9, Mem{Base: R10}.Offset(224))
+	MOVO(X10, Mem{Base: R10}.Offset(240))
+	MOVO(X11, Mem{Base: R10}.Offset(256))
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 10, 3, 7, 9, 14, 6, 1, 4)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 2, 5, 4, 15, 6, 10, 0, 8)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 14, 11, 6, 3, 1, 12, 8, 13)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 4, 7, 15, 1, 13, 5, 14, 9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 0, 6, 9, 8, 7, 3, 2, 11)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 5, 15, 8, 2, 0, 4, 6, 10)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 12, 13, 1, 10, 2, 7, 4, 5)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	LOAD_MSG(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	LOAD_MSG(X8, X9, X10, X11, SI, 15, 9, 3, 13, 11, 14, 12, 0)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(16), Mem{Base: R10}.Offset(32), Mem{Base: R10}.Offset(48), Mem{Base: R10}.Offset(64), X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(80), Mem{Base: R10}.Offset(96), Mem{Base: R10}.Offset(112), Mem{Base: R10}.Offset(128), X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(144), Mem{Base: R10}.Offset(160), Mem{Base: R10}.Offset(176), Mem{Base: R10}.Offset(192), X11, X13, X14)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, Mem{Base: R10}.Offset(208), Mem{Base: R10}.Offset(224), Mem{Base: R10}.Offset(240), Mem{Base: R10}.Offset(256), X11, X13, X14)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+
+	MOVOU(Mem{Base: AX}.Offset(32), X10)
+	MOVOU(Mem{Base: AX}.Offset(48), X11)
+	PXOR(X0, X12)
+	PXOR(X1, X15)
+	PXOR(X2, X10)
+	PXOR(X3, X11)
+	PXOR(X4, X12)
+	PXOR(X5, X15)
+	PXOR(X6, X10)
+	PXOR(X7, X11)
+	MOVOU(X10, Mem{Base: AX}.Offset(32))
+	MOVOU(X11, Mem{Base: AX}.Offset(48))
+
+	LEAQ(Mem{Base: SI}.Offset(128), RSI)
+	SUBQ(Imm(128), RDI)
+	JNE(LabelRef("loop"))
+
+	MOVOU(X12, Mem{Base: AX}.Offset(0))
+	MOVOU(X15, Mem{Base: AX}.Offset(16))
+
+	MOVQ(R8, Mem{Base: BX}.Offset(0))
+	MOVQ(R9, Mem{Base: BX}.Offset(8))
+
+	RET()
+}
+
+// #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
+
+func iv0_DATA() Mem {
+	if iv0_DATA_ptr != nil {
+		return *iv0_DATA_ptr
+	}
+
+	iv0 := GLOBL(ThatPeskyUnicodeDot+"iv0", NOPTR|RODATA)
+	iv0_DATA_ptr = &iv0
+	DATA(0x00, U64(0x6a09e667f3bcc908))
+	DATA(0x08, U64(0xbb67ae8584caa73b))
+	return iv0
+}
+
+func iv1_DATA() Mem {
+	if iv1_DATA_ptr != nil {
+		return *iv1_DATA_ptr
+	}
+
+	iv1 := GLOBL(ThatPeskyUnicodeDot+"iv1", NOPTR|RODATA)
+	iv1_DATA_ptr = &iv1
+	DATA(0x00, U64(0x3c6ef372fe94f82b))
+	DATA(0x08, U64(0xa54ff53a5f1d36f1))
+	return iv1
+}
+
+func iv2_DATA() Mem {
+	if iv2_DATA_ptr != nil {
+		return *iv2_DATA_ptr
+	}
+
+	iv2 := GLOBL(ThatPeskyUnicodeDot+"iv2", NOPTR|RODATA)
+	iv2_DATA_ptr = &iv2
+	DATA(0x00, U64(0x510e527fade682d1))
+	DATA(0x08, U64(0x9b05688c2b3e6c1f))
+	return iv2
+}
+
+func iv3_DATA() Mem {
+	if iv3_DATA_ptr != nil {
+		return *iv3_DATA_ptr
+	}
+
+	iv3 := GLOBL(ThatPeskyUnicodeDot+"iv3", NOPTR|RODATA)
+	iv3_DATA_ptr = &iv3
+	DATA(0x00, U64(0x1f83d9abfb41bd6b))
+	DATA(0x08, U64(0x5be0cd19137e2179))
+	return iv3
+}
+
+func c40_DATA() Mem {
+	if c40_DATA_ptr != nil {
+		return *c40_DATA_ptr
+	}
+
+	c40 := GLOBL(ThatPeskyUnicodeDot+"c40", NOPTR|RODATA)
+	c40_DATA_ptr = &c40
+	DATA(0x00, U64(0x0201000706050403))
+	DATA(0x08, U64(0x0a09080f0e0d0c0b))
+	return c40
+}
+
+func c48_DATA() Mem {
+	if c48_DATA_ptr != nil {
+		return *c48_DATA_ptr
+	}
+
+	c48 := GLOBL(ThatPeskyUnicodeDot+"c48", NOPTR|RODATA)
+	c48_DATA_ptr = &c48
+	DATA(0x00, U64(0x0100070605040302))
+	DATA(0x08, U64(0x09080f0e0d0c0b0a))
+	return c48
+}

blake2b/_asm/standard/go.mod

@@ -0,0 +1,15 @@
+module blake2b/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

blake2b/_asm/standard/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

blake2s/_asm/blake2s_amd64_asm.go

@@ -0,0 +1,525 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	"github.com/mmcloughlin/avo/ir"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/blake2s"
+)
+
+//go:generate go run . -out ../blake2s_amd64.s -pkg blake2s
+
+func main() {
+	Package("golang.org/x/crypto/blake2s")
+	ConstraintExpr("amd64,gc,!purego")
+	hashBlocksSSE2()
+	hashBlocksSSSE3()
+	hashBlocksSSE4()
+	Generate()
+}
+
+func ROTL_SSE2(n uint64, t, v VecPhysical) {
+	MOVO(v, t)
+	PSLLL(Imm(n), t)
+	PSRLL(Imm(32-n), v)
+	PXOR(t, v)
+}
+
+func ROTL_SSSE3(c, v VecPhysical) {
+	PSHUFB(c, v)
+}
+
+func ROUND_SSE2(v0, v1, v2, v3 VecPhysical, m0, m1, m2, m3 Mem, t VecPhysical) {
+	PADDL(m0, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(16, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m1, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(24, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v1, v1)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v3, v3)
+	PADDL(m2, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(16, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m3, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(24, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v3, v3)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v1, v1)
+}
+
+func ROUND_SSSE3(v0, v1, v2, v3 VecPhysical, m0, m1, m2, m3 Op, t, c16, c8 VecPhysical) {
+	PADDL(m0, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c16, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m1, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c8, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v1, v1)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v3, v3)
+	PADDL(m2, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c16, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m3, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c8, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v3, v3)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v1, v1)
+}
+
+func LOAD_MSG_SSE4(m0, m1, m2, m3 VecPhysical, src GPPhysical, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15 int) {
+	// Hack to get Avo to emit a MOVL instruction with a VecPhysical as the destination
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i0 * 4), m0}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i1*4), m0)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i2*4), m0)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i3*4), m0)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i4 * 4), m1}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i5*4), m1)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i6*4), m1)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i7*4), m1)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i8 * 4), m2}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i9*4), m2)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i10*4), m2)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i11*4), m2)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i12 * 4), m3}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i13*4), m3)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i14*4), m3)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i15*4), m3)
+}
+
+func PRECOMPUTE_MSG(dst GPPhysical, off int, src, R8, R9, R10, R11, R12, R13, R14, R15 GPPhysical) {
+	MOVQ(Mem{Base: src}.Offset(0*4), R8)
+	MOVQ(Mem{Base: src}.Offset(2*4), R9)
+	MOVQ(Mem{Base: src}.Offset(4*4), R10)
+	MOVQ(Mem{Base: src}.Offset(6*4), R11)
+	MOVQ(Mem{Base: src}.Offset(8*4), R12)
+	MOVQ(Mem{Base: src}.Offset(10*4), R13)
+	MOVQ(Mem{Base: src}.Offset(12*4), R14)
+	MOVQ(Mem{Base: src}.Offset(14*4), R15)
+
+	MOVL(R8L, Mem{Base: dst}.Offset(0*4+off+0))
+	MOVL(R8L, Mem{Base: dst}.Offset(9*4+off+64))
+	MOVL(R8L, Mem{Base: dst}.Offset(5*4+off+128))
+	MOVL(R8L, Mem{Base: dst}.Offset(14*4+off+192))
+	MOVL(R8L, Mem{Base: dst}.Offset(4*4+off+256))
+	MOVL(R8L, Mem{Base: dst}.Offset(2*4+off+320))
+	MOVL(R8L, Mem{Base: dst}.Offset(8*4+off+384))
+	MOVL(R8L, Mem{Base: dst}.Offset(12*4+off+448))
+	MOVL(R8L, Mem{Base: dst}.Offset(3*4+off+512))
+	MOVL(R8L, Mem{Base: dst}.Offset(15*4+off+576))
+	SHRQ(Imm(32), R8)
+	MOVL(R8L, Mem{Base: dst}.Offset(4*4+off+0))
+	MOVL(R8L, Mem{Base: dst}.Offset(8*4+off+64))
+	MOVL(R8L, Mem{Base: dst}.Offset(14*4+off+128))
+	MOVL(R8L, Mem{Base: dst}.Offset(5*4+off+192))
+	MOVL(R8L, Mem{Base: dst}.Offset(12*4+off+256))
+	MOVL(R8L, Mem{Base: dst}.Offset(11*4+off+320))
+	MOVL(R8L, Mem{Base: dst}.Offset(1*4+off+384))
+	MOVL(R8L, Mem{Base: dst}.Offset(6*4+off+448))
+	MOVL(R8L, Mem{Base: dst}.Offset(10*4+off+512))
+	MOVL(R8L, Mem{Base: dst}.Offset(3*4+off+576))
+
+	MOVL(R9L, Mem{Base: dst}.Offset(1*4+off+0))
+	MOVL(R9L, Mem{Base: dst}.Offset(13*4+off+64))
+	MOVL(R9L, Mem{Base: dst}.Offset(6*4+off+128))
+	MOVL(R9L, Mem{Base: dst}.Offset(8*4+off+192))
+	MOVL(R9L, Mem{Base: dst}.Offset(2*4+off+256))
+	MOVL(R9L, Mem{Base: dst}.Offset(0*4+off+320))
+	MOVL(R9L, Mem{Base: dst}.Offset(14*4+off+384))
+	MOVL(R9L, Mem{Base: dst}.Offset(11*4+off+448))
+	MOVL(R9L, Mem{Base: dst}.Offset(12*4+off+512))
+	MOVL(R9L, Mem{Base: dst}.Offset(4*4+off+576))
+	SHRQ(Imm(32), R9)
+	MOVL(R9L, Mem{Base: dst}.Offset(5*4+off+0))
+	MOVL(R9L, Mem{Base: dst}.Offset(15*4+off+64))
+	MOVL(R9L, Mem{Base: dst}.Offset(9*4+off+128))
+	MOVL(R9L, Mem{Base: dst}.Offset(1*4+off+192))
+	MOVL(R9L, Mem{Base: dst}.Offset(11*4+off+256))
+	MOVL(R9L, Mem{Base: dst}.Offset(7*4+off+320))
+	MOVL(R9L, Mem{Base: dst}.Offset(13*4+off+384))
+	MOVL(R9L, Mem{Base: dst}.Offset(3*4+off+448))
+	MOVL(R9L, Mem{Base: dst}.Offset(6*4+off+512))
+	MOVL(R9L, Mem{Base: dst}.Offset(10*4+off+576))
+
+	MOVL(R10L, Mem{Base: dst}.Offset(2*4+off+0))
+	MOVL(R10L, Mem{Base: dst}.Offset(1*4+off+64))
+	MOVL(R10L, Mem{Base: dst}.Offset(15*4+off+128))
+	MOVL(R10L, Mem{Base: dst}.Offset(10*4+off+192))
+	MOVL(R10L, Mem{Base: dst}.Offset(6*4+off+256))
+	MOVL(R10L, Mem{Base: dst}.Offset(8*4+off+320))
+	MOVL(R10L, Mem{Base: dst}.Offset(3*4+off+384))
+	MOVL(R10L, Mem{Base: dst}.Offset(13*4+off+448))
+	MOVL(R10L, Mem{Base: dst}.Offset(14*4+off+512))
+	MOVL(R10L, Mem{Base: dst}.Offset(5*4+off+576))
+	SHRQ(Imm(32), R10)
+	MOVL(R10L, Mem{Base: dst}.Offset(6*4+off+0))
+	MOVL(R10L, Mem{Base: dst}.Offset(11*4+off+64))
+	MOVL(R10L, Mem{Base: dst}.Offset(2*4+off+128))
+	MOVL(R10L, Mem{Base: dst}.Offset(9*4+off+192))
+	MOVL(R10L, Mem{Base: dst}.Offset(1*4+off+256))
+	MOVL(R10L, Mem{Base: dst}.Offset(13*4+off+320))
+	MOVL(R10L, Mem{Base: dst}.Offset(4*4+off+384))
+	MOVL(R10L, Mem{Base: dst}.Offset(8*4+off+448))
+	MOVL(R10L, Mem{Base: dst}.Offset(15*4+off+512))
+	MOVL(R10L, Mem{Base: dst}.Offset(7*4+off+576))
+
+	MOVL(R11L, Mem{Base: dst}.Offset(3*4+off+0))
+	MOVL(R11L, Mem{Base: dst}.Offset(7*4+off+64))
+	MOVL(R11L, Mem{Base: dst}.Offset(13*4+off+128))
+	MOVL(R11L, Mem{Base: dst}.Offset(12*4+off+192))
+	MOVL(R11L, Mem{Base: dst}.Offset(10*4+off+256))
+	MOVL(R11L, Mem{Base: dst}.Offset(1*4+off+320))
+	MOVL(R11L, Mem{Base: dst}.Offset(9*4+off+384))
+	MOVL(R11L, Mem{Base: dst}.Offset(14*4+off+448))
+	MOVL(R11L, Mem{Base: dst}.Offset(0*4+off+512))
+	MOVL(R11L, Mem{Base: dst}.Offset(6*4+off+576))
+	SHRQ(Imm(32), R11)
+	MOVL(R11L, Mem{Base: dst}.Offset(7*4+off+0))
+	MOVL(R11L, Mem{Base: dst}.Offset(14*4+off+64))
+	MOVL(R11L, Mem{Base: dst}.Offset(10*4+off+128))
+	MOVL(R11L, Mem{Base: dst}.Offset(0*4+off+192))
+	MOVL(R11L, Mem{Base: dst}.Offset(5*4+off+256))
+	MOVL(R11L, Mem{Base: dst}.Offset(9*4+off+320))
+	MOVL(R11L, Mem{Base: dst}.Offset(12*4+off+384))
+	MOVL(R11L, Mem{Base: dst}.Offset(1*4+off+448))
+	MOVL(R11L, Mem{Base: dst}.Offset(13*4+off+512))
+	MOVL(R11L, Mem{Base: dst}.Offset(2*4+off+576))
+
+	MOVL(R12L, Mem{Base: dst}.Offset(8*4+off+0))
+	MOVL(R12L, Mem{Base: dst}.Offset(5*4+off+64))
+	MOVL(R12L, Mem{Base: dst}.Offset(4*4+off+128))
+	MOVL(R12L, Mem{Base: dst}.Offset(15*4+off+192))
+	MOVL(R12L, Mem{Base: dst}.Offset(14*4+off+256))
+	MOVL(R12L, Mem{Base: dst}.Offset(3*4+off+320))
+	MOVL(R12L, Mem{Base: dst}.Offset(11*4+off+384))
+	MOVL(R12L, Mem{Base: dst}.Offset(10*4+off+448))
+	MOVL(R12L, Mem{Base: dst}.Offset(7*4+off+512))
+	MOVL(R12L, Mem{Base: dst}.Offset(1*4+off+576))
+	SHRQ(Imm(32), R12)
+	MOVL(R12L, Mem{Base: dst}.Offset(12*4+off+0))
+	MOVL(R12L, Mem{Base: dst}.Offset(2*4+off+64))
+	MOVL(R12L, Mem{Base: dst}.Offset(11*4+off+128))
+	MOVL(R12L, Mem{Base: dst}.Offset(4*4+off+192))
+	MOVL(R12L, Mem{Base: dst}.Offset(0*4+off+256))
+	MOVL(R12L, Mem{Base: dst}.Offset(15*4+off+320))
+	MOVL(R12L, Mem{Base: dst}.Offset(10*4+off+384))
+	MOVL(R12L, Mem{Base: dst}.Offset(7*4+off+448))
+	MOVL(R12L, Mem{Base: dst}.Offset(5*4+off+512))
+	MOVL(R12L, Mem{Base: dst}.Offset(9*4+off+576))
+
+	MOVL(R13L, Mem{Base: dst}.Offset(9*4+off+0))
+	MOVL(R13L, Mem{Base: dst}.Offset(4*4+off+64))
+	MOVL(R13L, Mem{Base: dst}.Offset(8*4+off+128))
+	MOVL(R13L, Mem{Base: dst}.Offset(13*4+off+192))
+	MOVL(R13L, Mem{Base: dst}.Offset(3*4+off+256))
+	MOVL(R13L, Mem{Base: dst}.Offset(5*4+off+320))
+	MOVL(R13L, Mem{Base: dst}.Offset(7*4+off+384))
+	MOVL(R13L, Mem{Base: dst}.Offset(15*4+off+448))
+	MOVL(R13L, Mem{Base: dst}.Offset(11*4+off+512))
+	MOVL(R13L, Mem{Base: dst}.Offset(0*4+off+576))
+	SHRQ(Imm(32), R13)
+	MOVL(R13L, Mem{Base: dst}.Offset(13*4+off+0))
+	MOVL(R13L, Mem{Base: dst}.Offset(10*4+off+64))
+	MOVL(R13L, Mem{Base: dst}.Offset(0*4+off+128))
+	MOVL(R13L, Mem{Base: dst}.Offset(3*4+off+192))
+	MOVL(R13L, Mem{Base: dst}.Offset(9*4+off+256))
+	MOVL(R13L, Mem{Base: dst}.Offset(6*4+off+320))
+	MOVL(R13L, Mem{Base: dst}.Offset(15*4+off+384))
+	MOVL(R13L, Mem{Base: dst}.Offset(4*4+off+448))
+	MOVL(R13L, Mem{Base: dst}.Offset(2*4+off+512))
+	MOVL(R13L, Mem{Base: dst}.Offset(12*4+off+576))
+
+	MOVL(R14L, Mem{Base: dst}.Offset(10*4+off+0))
+	MOVL(R14L, Mem{Base: dst}.Offset(12*4+off+64))
+	MOVL(R14L, Mem{Base: dst}.Offset(1*4+off+128))
+	MOVL(R14L, Mem{Base: dst}.Offset(6*4+off+192))
+	MOVL(R14L, Mem{Base: dst}.Offset(13*4+off+256))
+	MOVL(R14L, Mem{Base: dst}.Offset(4*4+off+320))
+	MOVL(R14L, Mem{Base: dst}.Offset(0*4+off+384))
+	MOVL(R14L, Mem{Base: dst}.Offset(2*4+off+448))
+	MOVL(R14L, Mem{Base: dst}.Offset(8*4+off+512))
+	MOVL(R14L, Mem{Base: dst}.Offset(14*4+off+576))
+	SHRQ(Imm(32), R14)
+	MOVL(R14L, Mem{Base: dst}.Offset(14*4+off+0))
+	MOVL(R14L, Mem{Base: dst}.Offset(3*4+off+64))
+	MOVL(R14L, Mem{Base: dst}.Offset(7*4+off+128))
+	MOVL(R14L, Mem{Base: dst}.Offset(2*4+off+192))
+	MOVL(R14L, Mem{Base: dst}.Offset(15*4+off+256))
+	MOVL(R14L, Mem{Base: dst}.Offset(12*4+off+320))
+	MOVL(R14L, Mem{Base: dst}.Offset(6*4+off+384))
+	MOVL(R14L, Mem{Base: dst}.Offset(0*4+off+448))
+	MOVL(R14L, Mem{Base: dst}.Offset(9*4+off+512))
+	MOVL(R14L, Mem{Base: dst}.Offset(11*4+off+576))
+
+	MOVL(R15L, Mem{Base: dst}.Offset(11*4+off+0))
+	MOVL(R15L, Mem{Base: dst}.Offset(0*4+off+64))
+	MOVL(R15L, Mem{Base: dst}.Offset(12*4+off+128))
+	MOVL(R15L, Mem{Base: dst}.Offset(7*4+off+192))
+	MOVL(R15L, Mem{Base: dst}.Offset(8*4+off+256))
+	MOVL(R15L, Mem{Base: dst}.Offset(14*4+off+320))
+	MOVL(R15L, Mem{Base: dst}.Offset(2*4+off+384))
+	MOVL(R15L, Mem{Base: dst}.Offset(5*4+off+448))
+	MOVL(R15L, Mem{Base: dst}.Offset(1*4+off+512))
+	MOVL(R15L, Mem{Base: dst}.Offset(13*4+off+576))
+	SHRQ(Imm(32), R15)
+	MOVL(R15L, Mem{Base: dst}.Offset(15*4+off+0))
+	MOVL(R15L, Mem{Base: dst}.Offset(6*4+off+64))
+	MOVL(R15L, Mem{Base: dst}.Offset(3*4+off+128))
+	MOVL(R15L, Mem{Base: dst}.Offset(11*4+off+192))
+	MOVL(R15L, Mem{Base: dst}.Offset(7*4+off+256))
+	MOVL(R15L, Mem{Base: dst}.Offset(10*4+off+320))
+	MOVL(R15L, Mem{Base: dst}.Offset(5*4+off+384))
+	MOVL(R15L, Mem{Base: dst}.Offset(9*4+off+448))
+	MOVL(R15L, Mem{Base: dst}.Offset(4*4+off+512))
+	MOVL(R15L, Mem{Base: dst}.Offset(8*4+off+576))
+}
+
+func BLAKE2s_SSE2() {
+	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15)
+	for i := 0; i < 10; i++ {
+		ROUND_SSE2(X4, X5, X6, X7, Mem{Base: BP}.Offset(16+64*i), Mem{Base: BP}.Offset(32+64*i), Mem{Base: BP}.Offset(48+64*i), Mem{Base: BP}.Offset(64+64*i), X8)
+	}
+}
+
+func BLAKE2s_SSSE3() {
+	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15)
+	for i := 0; i < 10; i++ {
+		ROUND_SSSE3(X4, X5, X6, X7, Mem{Base: BP}.Offset(16+64*i), Mem{Base: BP}.Offset(32+64*i), Mem{Base: BP}.Offset(48+64*i), Mem{Base: BP}.Offset(64+64*i), X8, X13, X14)
+	}
+}
+
+func BLAKE2s_SSE4() {
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+}
+
+func HASH_BLOCKS(h, c, flag, blocks_base, blocks_len Mem, BLAKE2s_FUNC func()) {
+	MOVQ(h, RAX)
+	MOVQ(c, RBX)
+	MOVL(flag, ECX)
+	MOVQ(blocks_base, RSI)
+	MOVQ(blocks_len, RDX)
+
+	MOVQ(RSP, RBP)
+	ADDQ(Imm(15), RBP)
+	ANDQ(I32(^15), RBP)
+
+	MOVQ(Mem{Base: BX}.Offset(0), R9)
+	MOVQ(R9, Mem{Base: BP}.Offset(0))
+	MOVQ(RCX, Mem{Base: BP}.Offset(8))
+
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: AX}.Offset(16), X1)
+
+	iv0 := iv0_DATA()
+	iv1 := iv1_DATA()
+	MOVOU(iv0, X2)
+	MOVOU(iv1, X3)
+
+	counter := counter_DATA()
+	rol16 := rol16_DATA()
+	rol8 := rol8_DATA()
+	MOVOU(counter, X12)
+	MOVOU(rol16, X13)
+	MOVOU(rol8, X14)
+	MOVO(Mem{Base: BP}.Offset(0), X15)
+
+	Label("loop")
+	MOVO(X0, X4)
+	MOVO(X1, X5)
+	MOVO(X2, X6)
+	MOVO(X3, X7)
+
+	PADDQ(X12, X15)
+	PXOR(X15, X7)
+
+	BLAKE2s_FUNC()
+
+	PXOR(X4, X0)
+	PXOR(X5, X1)
+	PXOR(X6, X0)
+	PXOR(X7, X1)
+
+	LEAQ(Mem{Base: SI}.Offset(64), RSI)
+	SUBQ(Imm(64), RDX)
+	JNE(LabelRef("loop"))
+
+	MOVO(X15, Mem{Base: BP}.Offset(0))
+	MOVQ(Mem{Base: BP}.Offset(0), R9)
+	MOVQ(R9, Mem{Base: BX}.Offset(0))
+
+	MOVOU(X0, Mem{Base: AX}.Offset(0))
+	MOVOU(X1, Mem{Base: AX}.Offset(16))
+}
+
+func hashBlocksSSE2() {
+	Implement("hashBlocksSSE2")
+	Attributes(0)
+	AllocLocal(672) // frame = 656 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSE2)
+	RET()
+}
+
+func hashBlocksSSSE3() {
+	Implement("hashBlocksSSSE3")
+	Attributes(0)
+	AllocLocal(672) // frame = 656 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSSE3)
+	RET()
+}
+
+func hashBlocksSSE4() {
+	Implement("hashBlocksSSE4")
+	Attributes(0)
+	AllocLocal(32) // frame = 16 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSE4)
+	RET()
+}
+
+// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
+
+var iv0_DATA_ptr, iv1_DATA_ptr, rol16_DATA_ptr, rol8_DATA_ptr, counter_DATA_ptr *Mem
+
+func iv0_DATA() Mem {
+	if iv0_DATA_ptr != nil {
+		return *iv0_DATA_ptr
+	}
+
+	iv0_DATA := GLOBL("iv0", NOPTR|RODATA)
+	iv0_DATA_ptr = &iv0_DATA
+	DATA(0x00, U32(0x6a09e667))
+	DATA(0x04, U32(0xbb67ae85))
+	DATA(0x08, U32(0x3c6ef372))
+	DATA(0x0c, U32(0xa54ff53a))
+	return iv0_DATA
+}
+
+func iv1_DATA() Mem {
+	if iv1_DATA_ptr != nil {
+		return *iv1_DATA_ptr
+	}
+
+	iv1_DATA := GLOBL("iv1", NOPTR|RODATA)
+	iv1_DATA_ptr = &iv1_DATA
+	DATA(0x00, U32(0x510e527f))
+	DATA(0x04, U32(0x9b05688c))
+	DATA(0x08, U32(0x1f83d9ab))
+	DATA(0x0c, U32(0x5be0cd19))
+	return iv1_DATA
+}
+
+func rol16_DATA() Mem {
+	if rol16_DATA_ptr != nil {
+		return *rol16_DATA_ptr
+	}
+
+	rol16_DATA := GLOBL("rol16", NOPTR|RODATA)
+	rol16_DATA_ptr = &rol16_DATA
+	DATA(0x00, U64(0x0504070601000302))
+	DATA(0x08, U64(0x0D0C0F0E09080B0A))
+	return rol16_DATA
+}
+
+func rol8_DATA() Mem {
+	if rol8_DATA_ptr != nil {
+		return *rol8_DATA_ptr
+	}
+
+	rol8_DATA := GLOBL("rol8", NOPTR|RODATA)
+	rol8_DATA_ptr = &rol8_DATA
+	DATA(0x00, U64(0x0407060500030201))
+	DATA(0x08, U64(0x0C0F0E0D080B0A09))
+	return rol8_DATA
+}
+
+func counter_DATA() Mem {
+	if counter_DATA_ptr != nil {
+		return *counter_DATA_ptr
+	}
+
+	counter_DATA := GLOBL("counter", NOPTR|RODATA)
+	counter_DATA_ptr = &counter_DATA
+	DATA(0x00, U64(0x0000000000000040))
+	DATA(0x08, U64(0x0000000000000000))
+	return counter_DATA
+}

blake2s/_asm/go.mod

@@ -0,0 +1,15 @@
+module blake2s/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

blake2s/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

blake2s/blake2s.go

@@ -16,9 +16,10 @@
 //
 // BLAKE2X is a construction to compute hash values larger than 32 bytes. It
 // can produce hash values between 0 and 65535 bytes.
-package blake2s // import "golang.org/x/crypto/blake2s"
+package blake2s
 
 import (
+	"crypto"
 	"encoding/binary"
 	"errors"
 	"hash"
@@ -55,6 +56,13 @@ func Sum256(data []byte) [Size]byte {
 // and BinaryUnmarshaler for state (de)serialization as documented by hash.Hash.
 func New256(key []byte) (hash.Hash, error) { return newDigest(Size, key) }
 
+func init() {
+	crypto.RegisterHash(crypto.BLAKE2s_256, func() hash.Hash {
+		h, _ := New256(nil)
+		return h
+	})
+}
+
 // New128 returns a new hash.Hash computing the BLAKE2s-128 checksum given a
 // non-empty key. Note that a 128-bit digest is too small to be secure as a
 // cryptographic hash and should only be used as a MAC, thus the key argument

blake2s/register.go

@@ -1,21 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build go1.9
-
-package blake2s
-
-import (
-	"crypto"
-	"hash"
-)
-
-func init() {
-	newHash256 := func() hash.Hash {
-		h, _ := New256(nil)
-		return h
-	}
-
-	crypto.RegisterHash(crypto.BLAKE2s_256, newHash256)
-}

blowfish/cipher.go

@@ -11,7 +11,7 @@
 // Deprecated: any new system should use AES (from crypto/aes, if necessary in
 // an AEAD mode like crypto/cipher.NewGCM) or XChaCha20-Poly1305 (from
 // golang.org/x/crypto/chacha20poly1305).
-package blowfish // import "golang.org/x/crypto/blowfish"
+package blowfish
 
 // The code is a port of Bruce Schneier's C implementation.
 // See https://www.schneier.com/blowfish.html.

bn256/bn256.go

@@ -23,7 +23,7 @@
 // elliptic curve. This package is frozen, and not implemented in constant time.
 // There is a more complete implementation at github.com/cloudflare/bn256, but
 // note that it suffers from the same security issues of the underlying curve.
-package bn256 // import "golang.org/x/crypto/bn256"
+package bn256
 
 import (
 	"crypto/rand"

cast5/cast5.go

@@ -11,7 +11,7 @@
 // Deprecated: any new system should use AES (from crypto/aes, if necessary in
 // an AEAD mode like crypto/cipher.NewGCM) or XChaCha20-Poly1305 (from
 // golang.org/x/crypto/chacha20poly1305).
-package cast5 // import "golang.org/x/crypto/cast5"
+package cast5
 
 import (
 	"errors"

chacha20poly1305/_asm/go.mod

@@ -0,0 +1,15 @@
+module chacha20poly1305/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

chacha20poly1305/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

chacha20poly1305/chacha20poly1305.go

@@ -5,7 +5,7 @@
 // Package chacha20poly1305 implements the ChaCha20-Poly1305 AEAD and its
 // extended nonce variant XChaCha20-Poly1305, as specified in RFC 8439 and
 // draft-irtf-cfrg-xchacha-01.
-package chacha20poly1305 // import "golang.org/x/crypto/chacha20poly1305"
+package chacha20poly1305
 
 import (
 	"crypto/cipher"

cryptobyte/asn1/asn1.go

@@ -4,7 +4,7 @@
 
 // Package asn1 contains supporting types for parsing and building ASN.1
 // messages with the cryptobyte package.
-package asn1 // import "golang.org/x/crypto/cryptobyte/asn1"
+package asn1
 
 // Tag represents an ASN.1 identifier octet, consisting of a tag number
 // (indicating a type) and class (such as context-specific or constructed).

cryptobyte/string.go

@@ -15,7 +15,7 @@
 //
 // See the documentation and examples for the Builder and String types to get
 // started.
-package cryptobyte // import "golang.org/x/crypto/cryptobyte"
+package cryptobyte
 
 // String represents a string of bytes. It provides methods for parsing
 // fixed-length and length-prefixed values from it.

curve25519/curve25519.go

@@ -6,9 +6,11 @@
 // performs scalar multiplication on the elliptic curve known as Curve25519.
 // See RFC 7748.
 //
-// Starting in Go 1.20, this package is a wrapper for the X25519 implementation
+// This package is a wrapper for the X25519 implementation
 // in the crypto/ecdh package.
-package curve25519 // import "golang.org/x/crypto/curve25519"
+package curve25519
+
+import "crypto/ecdh"
 
 // ScalarMult sets dst to the product scalar * point.
 //
@@ -16,7 +18,13 @@ package curve25519 // import "golang.org/x/crypto/curve25519"
 // zeroes, irrespective of the scalar. Instead, use the X25519 function, which
 // will return an error.
 func ScalarMult(dst, scalar, point *[32]byte) {
-	scalarMult(dst, scalar, point)
+	if _, err := x25519(dst, scalar[:], point[:]); err != nil {
+		// The only error condition for x25519 when the inputs are 32 bytes long
+		// is if the output would have been the all-zero value.
+		for i := range dst {
+			dst[i] = 0
+		}
+	}
 }
 
 // ScalarBaseMult sets dst to the product scalar * base where base is the
@@ -25,7 +33,12 @@ func ScalarMult(dst, scalar, point *[32]byte) {
 // It is recommended to use the X25519 function with Basepoint instead, as
 // copying into fixed size arrays can lead to unexpected bugs.
 func ScalarBaseMult(dst, scalar *[32]byte) {
-	scalarBaseMult(dst, scalar)
+	curve := ecdh.X25519()
+	priv, err := curve.NewPrivateKey(scalar[:])
+	if err != nil {
+		panic("curve25519: internal error: scalarBaseMult was not 32 bytes")
+	}
+	copy(dst[:], priv.PublicKey().Bytes())
 }
 
 const (
@@ -57,3 +70,21 @@ func X25519(scalar, point []byte) ([]byte, error) {
 	var dst [32]byte
 	return x25519(&dst, scalar, point)
 }
+
+func x25519(dst *[32]byte, scalar, point []byte) ([]byte, error) {
+	curve := ecdh.X25519()
+	pub, err := curve.NewPublicKey(point)
+	if err != nil {
+		return nil, err
+	}
+	priv, err := curve.NewPrivateKey(scalar)
+	if err != nil {
+		return nil, err
+	}
+	out, err := priv.ECDH(pub)
+	if err != nil {
+		return nil, err
+	}
+	copy(dst[:], out)
+	return dst[:], nil
+}

curve25519/curve25519_compat.go

@@ -1,105 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build !go1.20
-
-package curve25519
-
-import (
-	"crypto/subtle"
-	"errors"
-	"strconv"
-
-	"golang.org/x/crypto/curve25519/internal/field"
-)
-
-func scalarMult(dst, scalar, point *[32]byte) {
-	var e [32]byte
-
-	copy(e[:], scalar[:])
-	e[0] &= 248
-	e[31] &= 127
-	e[31] |= 64
-
-	var x1, x2, z2, x3, z3, tmp0, tmp1 field.Element
-	x1.SetBytes(point[:])
-	x2.One()
-	x3.Set(&x1)
-	z3.One()
-
-	swap := 0
-	for pos := 254; pos >= 0; pos-- {
-		b := e[pos/8] >> uint(pos&7)
-		b &= 1
-		swap ^= int(b)
-		x2.Swap(&x3, swap)
-		z2.Swap(&z3, swap)
-		swap = int(b)
-
-		tmp0.Subtract(&x3, &z3)
-		tmp1.Subtract(&x2, &z2)
-		x2.Add(&x2, &z2)
-		z2.Add(&x3, &z3)
-		z3.Multiply(&tmp0, &x2)
-		z2.Multiply(&z2, &tmp1)
-		tmp0.Square(&tmp1)
-		tmp1.Square(&x2)
-		x3.Add(&z3, &z2)
-		z2.Subtract(&z3, &z2)
-		x2.Multiply(&tmp1, &tmp0)
-		tmp1.Subtract(&tmp1, &tmp0)
-		z2.Square(&z2)
-
-		z3.Mult32(&tmp1, 121666)
-		x3.Square(&x3)
-		tmp0.Add(&tmp0, &z3)
-		z3.Multiply(&x1, &z2)
-		z2.Multiply(&tmp1, &tmp0)
-	}
-
-	x2.Swap(&x3, swap)
-	z2.Swap(&z3, swap)
-
-	z2.Invert(&z2)
-	x2.Multiply(&x2, &z2)
-	copy(dst[:], x2.Bytes())
-}
-
-func scalarBaseMult(dst, scalar *[32]byte) {
-	checkBasepoint()
-	scalarMult(dst, scalar, &basePoint)
-}
-
-func x25519(dst *[32]byte, scalar, point []byte) ([]byte, error) {
-	var in [32]byte
-	if l := len(scalar); l != 32 {
-		return nil, errors.New("bad scalar length: " + strconv.Itoa(l) + ", expected 32")
-	}
-	if l := len(point); l != 32 {
-		return nil, errors.New("bad point length: " + strconv.Itoa(l) + ", expected 32")
-	}
-	copy(in[:], scalar)
-	if &point[0] == &Basepoint[0] {
-		scalarBaseMult(dst, &in)
-	} else {
-		var base, zero [32]byte
-		copy(base[:], point)
-		scalarMult(dst, &in, &base)
-		if subtle.ConstantTimeCompare(dst[:], zero[:]) == 1 {
-			return nil, errors.New("bad input point: low order point")
-		}
-	}
-	return dst[:], nil
-}
-
-func checkBasepoint() {
-	if subtle.ConstantTimeCompare(Basepoint, []byte{
-		0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	}) != 1 {
-		panic("curve25519: global Basepoint value was modified")
-	}
-}

curve25519/curve25519_go120.go

@@ -1,46 +0,0 @@
-// Copyright 2022 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build go1.20
-
-package curve25519
-
-import "crypto/ecdh"
-
-func x25519(dst *[32]byte, scalar, point []byte) ([]byte, error) {
-	curve := ecdh.X25519()
-	pub, err := curve.NewPublicKey(point)
-	if err != nil {
-		return nil, err
-	}
-	priv, err := curve.NewPrivateKey(scalar)
-	if err != nil {
-		return nil, err
-	}
-	out, err := priv.ECDH(pub)
-	if err != nil {
-		return nil, err
-	}
-	copy(dst[:], out)
-	return dst[:], nil
-}
-
-func scalarMult(dst, scalar, point *[32]byte) {
-	if _, err := x25519(dst, scalar[:], point[:]); err != nil {
-		// The only error condition for x25519 when the inputs are 32 bytes long
-		// is if the output would have been the all-zero value.
-		for i := range dst {
-			dst[i] = 0
-		}
-	}
-}
-
-func scalarBaseMult(dst, scalar *[32]byte) {
-	curve := ecdh.X25519()
-	priv, err := curve.NewPrivateKey(scalar[:])
-	if err != nil {
-		panic("curve25519: internal error: scalarBaseMult was not 32 bytes")
-	}
-	copy(dst[:], priv.PublicKey().Bytes())
-}

curve25519/internal/field/README

@@ -1,7 +0,0 @@
-This package is kept in sync with crypto/ed25519/internal/edwards25519/field in
-the standard library.
-
-If there are any changes in the standard library that need to be synced to this
-package, run sync.sh. It will not overwrite any local changes made since the
-previous sync, so it's ok to land changes in this package first, and then sync
-to the standard library later.

curve25519/internal/field/_asm/fe_amd64_asm.go

@@ -1,298 +0,0 @@
-// Copyright (c) 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"fmt"
-
-	. "github.com/mmcloughlin/avo/build"
-	. "github.com/mmcloughlin/avo/gotypes"
-	. "github.com/mmcloughlin/avo/operand"
-	. "github.com/mmcloughlin/avo/reg"
-
-	// Ensure "go mod tidy" doesn't remove the golang.org/x/crypto module
-	// dependency, which is necessary to access the field.Element type.
-	_ "golang.org/x/crypto/curve25519"
-)
-
-//go:generate go run . -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field
-
-func main() {
-	Package("golang.org/x/crypto/curve25519/internal/field")
-	ConstraintExpr("amd64,gc,!purego")
-	feMul()
-	feSquare()
-	Generate()
-}
-
-type namedComponent struct {
-	Component
-	name string
-}
-
-func (c namedComponent) String() string { return c.name }
-
-type uint128 struct {
-	name   string
-	hi, lo GPVirtual
-}
-
-func (c uint128) String() string { return c.name }
-
-func feSquare() {
-	TEXT("feSquare", NOSPLIT, "func(out, a *Element)")
-	Doc("feSquare sets out = a * a. It works like feSquareGeneric.")
-	Pragma("noescape")
-
-	a := Dereference(Param("a"))
-	l0 := namedComponent{a.Field("l0"), "l0"}
-	l1 := namedComponent{a.Field("l1"), "l1"}
-	l2 := namedComponent{a.Field("l2"), "l2"}
-	l3 := namedComponent{a.Field("l3"), "l3"}
-	l4 := namedComponent{a.Field("l4"), "l4"}
-
-	// r0 = l0×l0 + 19×2×(l1×l4 + l2×l3)
-	r0 := uint128{"r0", GP64(), GP64()}
-	mul64(r0, 1, l0, l0)
-	addMul64(r0, 38, l1, l4)
-	addMul64(r0, 38, l2, l3)
-
-	// r1 = 2×l0×l1 + 19×2×l2×l4 + 19×l3×l3
-	r1 := uint128{"r1", GP64(), GP64()}
-	mul64(r1, 2, l0, l1)
-	addMul64(r1, 38, l2, l4)
-	addMul64(r1, 19, l3, l3)
-
-	// r2 = = 2×l0×l2 + l1×l1 + 19×2×l3×l4
-	r2 := uint128{"r2", GP64(), GP64()}
-	mul64(r2, 2, l0, l2)
-	addMul64(r2, 1, l1, l1)
-	addMul64(r2, 38, l3, l4)
-
-	// r3 = = 2×l0×l3 + 2×l1×l2 + 19×l4×l4
-	r3 := uint128{"r3", GP64(), GP64()}
-	mul64(r3, 2, l0, l3)
-	addMul64(r3, 2, l1, l2)
-	addMul64(r3, 19, l4, l4)
-
-	// r4 = = 2×l0×l4 + 2×l1×l3 + l2×l2
-	r4 := uint128{"r4", GP64(), GP64()}
-	mul64(r4, 2, l0, l4)
-	addMul64(r4, 2, l1, l3)
-	addMul64(r4, 1, l2, l2)
-
-	Comment("First reduction chain")
-	maskLow51Bits := GP64()
-	MOVQ(Imm((1<<51)-1), maskLow51Bits)
-	c0, r0lo := shiftRightBy51(&r0)
-	c1, r1lo := shiftRightBy51(&r1)
-	c2, r2lo := shiftRightBy51(&r2)
-	c3, r3lo := shiftRightBy51(&r3)
-	c4, r4lo := shiftRightBy51(&r4)
-	maskAndAdd(r0lo, maskLow51Bits, c4, 19)
-	maskAndAdd(r1lo, maskLow51Bits, c0, 1)
-	maskAndAdd(r2lo, maskLow51Bits, c1, 1)
-	maskAndAdd(r3lo, maskLow51Bits, c2, 1)
-	maskAndAdd(r4lo, maskLow51Bits, c3, 1)
-
-	Comment("Second reduction chain (carryPropagate)")
-	// c0 = r0 >> 51
-	MOVQ(r0lo, c0)
-	SHRQ(Imm(51), c0)
-	// c1 = r1 >> 51
-	MOVQ(r1lo, c1)
-	SHRQ(Imm(51), c1)
-	// c2 = r2 >> 51
-	MOVQ(r2lo, c2)
-	SHRQ(Imm(51), c2)
-	// c3 = r3 >> 51
-	MOVQ(r3lo, c3)
-	SHRQ(Imm(51), c3)
-	// c4 = r4 >> 51
-	MOVQ(r4lo, c4)
-	SHRQ(Imm(51), c4)
-	maskAndAdd(r0lo, maskLow51Bits, c4, 19)
-	maskAndAdd(r1lo, maskLow51Bits, c0, 1)
-	maskAndAdd(r2lo, maskLow51Bits, c1, 1)
-	maskAndAdd(r3lo, maskLow51Bits, c2, 1)
-	maskAndAdd(r4lo, maskLow51Bits, c3, 1)
-
-	Comment("Store output")
-	out := Dereference(Param("out"))
-	Store(r0lo, out.Field("l0"))
-	Store(r1lo, out.Field("l1"))
-	Store(r2lo, out.Field("l2"))
-	Store(r3lo, out.Field("l3"))
-	Store(r4lo, out.Field("l4"))
-
-	RET()
-}
-
-func feMul() {
-	TEXT("feMul", NOSPLIT, "func(out, a, b *Element)")
-	Doc("feMul sets out = a * b. It works like feMulGeneric.")
-	Pragma("noescape")
-
-	a := Dereference(Param("a"))
-	a0 := namedComponent{a.Field("l0"), "a0"}
-	a1 := namedComponent{a.Field("l1"), "a1"}
-	a2 := namedComponent{a.Field("l2"), "a2"}
-	a3 := namedComponent{a.Field("l3"), "a3"}
-	a4 := namedComponent{a.Field("l4"), "a4"}
-
-	b := Dereference(Param("b"))
-	b0 := namedComponent{b.Field("l0"), "b0"}
-	b1 := namedComponent{b.Field("l1"), "b1"}
-	b2 := namedComponent{b.Field("l2"), "b2"}
-	b3 := namedComponent{b.Field("l3"), "b3"}
-	b4 := namedComponent{b.Field("l4"), "b4"}
-
-	// r0 = a0×b0 + 19×(a1×b4 + a2×b3 + a3×b2 + a4×b1)
-	r0 := uint128{"r0", GP64(), GP64()}
-	mul64(r0, 1, a0, b0)
-	addMul64(r0, 19, a1, b4)
-	addMul64(r0, 19, a2, b3)
-	addMul64(r0, 19, a3, b2)
-	addMul64(r0, 19, a4, b1)
-
-	// r1 = a0×b1 + a1×b0 + 19×(a2×b4 + a3×b3 + a4×b2)
-	r1 := uint128{"r1", GP64(), GP64()}
-	mul64(r1, 1, a0, b1)
-	addMul64(r1, 1, a1, b0)
-	addMul64(r1, 19, a2, b4)
-	addMul64(r1, 19, a3, b3)
-	addMul64(r1, 19, a4, b2)
-
-	// r2 = a0×b2 + a1×b1 + a2×b0 + 19×(a3×b4 + a4×b3)
-	r2 := uint128{"r2", GP64(), GP64()}
-	mul64(r2, 1, a0, b2)
-	addMul64(r2, 1, a1, b1)
-	addMul64(r2, 1, a2, b0)
-	addMul64(r2, 19, a3, b4)
-	addMul64(r2, 19, a4, b3)
-
-	// r3 = a0×b3 + a1×b2 + a2×b1 + a3×b0 + 19×a4×b4
-	r3 := uint128{"r3", GP64(), GP64()}
-	mul64(r3, 1, a0, b3)
-	addMul64(r3, 1, a1, b2)
-	addMul64(r3, 1, a2, b1)
-	addMul64(r3, 1, a3, b0)
-	addMul64(r3, 19, a4, b4)
-
-	// r4 = a0×b4 + a1×b3 + a2×b2 + a3×b1 + a4×b0
-	r4 := uint128{"r4", GP64(), GP64()}
-	mul64(r4, 1, a0, b4)
-	addMul64(r4, 1, a1, b3)
-	addMul64(r4, 1, a2, b2)
-	addMul64(r4, 1, a3, b1)
-	addMul64(r4, 1, a4, b0)
-
-	Comment("First reduction chain")
-	maskLow51Bits := GP64()
-	MOVQ(Imm((1<<51)-1), maskLow51Bits)
-	c0, r0lo := shiftRightBy51(&r0)
-	c1, r1lo := shiftRightBy51(&r1)
-	c2, r2lo := shiftRightBy51(&r2)
-	c3, r3lo := shiftRightBy51(&r3)
-	c4, r4lo := shiftRightBy51(&r4)
-	maskAndAdd(r0lo, maskLow51Bits, c4, 19)
-	maskAndAdd(r1lo, maskLow51Bits, c0, 1)
-	maskAndAdd(r2lo, maskLow51Bits, c1, 1)
-	maskAndAdd(r3lo, maskLow51Bits, c2, 1)
-	maskAndAdd(r4lo, maskLow51Bits, c3, 1)
-
-	Comment("Second reduction chain (carryPropagate)")
-	// c0 = r0 >> 51
-	MOVQ(r0lo, c0)
-	SHRQ(Imm(51), c0)
-	// c1 = r1 >> 51
-	MOVQ(r1lo, c1)
-	SHRQ(Imm(51), c1)
-	// c2 = r2 >> 51
-	MOVQ(r2lo, c2)
-	SHRQ(Imm(51), c2)
-	// c3 = r3 >> 51
-	MOVQ(r3lo, c3)
-	SHRQ(Imm(51), c3)
-	// c4 = r4 >> 51
-	MOVQ(r4lo, c4)
-	SHRQ(Imm(51), c4)
-	maskAndAdd(r0lo, maskLow51Bits, c4, 19)
-	maskAndAdd(r1lo, maskLow51Bits, c0, 1)
-	maskAndAdd(r2lo, maskLow51Bits, c1, 1)
-	maskAndAdd(r3lo, maskLow51Bits, c2, 1)
-	maskAndAdd(r4lo, maskLow51Bits, c3, 1)
-
-	Comment("Store output")
-	out := Dereference(Param("out"))
-	Store(r0lo, out.Field("l0"))
-	Store(r1lo, out.Field("l1"))
-	Store(r2lo, out.Field("l2"))
-	Store(r3lo, out.Field("l3"))
-	Store(r4lo, out.Field("l4"))
-
-	RET()
-}
-
-// mul64 sets r to i * aX * bX.
-func mul64(r uint128, i int, aX, bX namedComponent) {
-	switch i {
-	case 1:
-		Comment(fmt.Sprintf("%s = %s×%s", r, aX, bX))
-		Load(aX, RAX)
-	case 2:
-		Comment(fmt.Sprintf("%s = 2×%s×%s", r, aX, bX))
-		Load(aX, RAX)
-		SHLQ(Imm(1), RAX)
-	default:
-		panic("unsupported i value")
-	}
-	MULQ(mustAddr(bX)) // RDX, RAX = RAX * bX
-	MOVQ(RAX, r.lo)
-	MOVQ(RDX, r.hi)
-}
-
-// addMul64 sets r to r + i * aX * bX.
-func addMul64(r uint128, i uint64, aX, bX namedComponent) {
-	switch i {
-	case 1:
-		Comment(fmt.Sprintf("%s += %s×%s", r, aX, bX))
-		Load(aX, RAX)
-	default:
-		Comment(fmt.Sprintf("%s += %d×%s×%s", r, i, aX, bX))
-		IMUL3Q(Imm(i), Load(aX, GP64()), RAX)
-	}
-	MULQ(mustAddr(bX)) // RDX, RAX = RAX * bX
-	ADDQ(RAX, r.lo)
-	ADCQ(RDX, r.hi)
-}
-
-// shiftRightBy51 returns r >> 51 and r.lo.
-//
-// After this function is called, the uint128 may not be used anymore.
-func shiftRightBy51(r *uint128) (out, lo GPVirtual) {
-	out = r.hi
-	lo = r.lo
-	SHLQ(Imm(64-51), r.lo, r.hi)
-	r.lo, r.hi = nil, nil // make sure the uint128 is unusable
-	return
-}
-
-// maskAndAdd sets r = r&mask + c*i.
-func maskAndAdd(r, mask, c GPVirtual, i uint64) {
-	ANDQ(mask, r)
-	if i != 1 {
-		IMUL3Q(Imm(i), c, c)
-	}
-	ADDQ(c, r)
-}
-
-func mustAddr(c Component) Op {
-	b, err := c.Resolve()
-	if err != nil {
-		panic(err)
-	}
-	return b.Addr
-}

curve25519/internal/field/_asm/go.mod

@@ -1,16 +0,0 @@
-module asm
-
-go 1.18
-
-require (
-	github.com/mmcloughlin/avo v0.5.0
-	golang.org/x/crypto v0.1.0
-)
-
-require (
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-)
-
-replace golang.org/x/crypto v0.1.0 => ../../../..

curve25519/internal/field/_asm/go.sum

@@ -1,51 +0,0 @@
-github.com/mmcloughlin/avo v0.5.0 h1:nAco9/aI9Lg2kiuROBY6BhCI/z0t5jEvJfjWbL8qXLU=
-github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
-golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

curve25519/internal/field/fe.go

@@ -1,416 +0,0 @@
-// Copyright (c) 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package field implements fast arithmetic modulo 2^255-19.
-package field
-
-import (
-	"crypto/subtle"
-	"encoding/binary"
-	"math/bits"
-)
-
-// Element represents an element of the field GF(2^255-19). Note that this
-// is not a cryptographically secure group, and should only be used to interact
-// with edwards25519.Point coordinates.
-//
-// This type works similarly to math/big.Int, and all arguments and receivers
-// are allowed to alias.
-//
-// The zero value is a valid zero element.
-type Element struct {
-	// An element t represents the integer
-	//     t.l0 + t.l1*2^51 + t.l2*2^102 + t.l3*2^153 + t.l4*2^204
-	//
-	// Between operations, all limbs are expected to be lower than 2^52.
-	l0 uint64
-	l1 uint64
-	l2 uint64
-	l3 uint64
-	l4 uint64
-}
-
-const maskLow51Bits uint64 = (1 << 51) - 1
-
-var feZero = &Element{0, 0, 0, 0, 0}
-
-// Zero sets v = 0, and returns v.
-func (v *Element) Zero() *Element {
-	*v = *feZero
-	return v
-}
-
-var feOne = &Element{1, 0, 0, 0, 0}
-
-// One sets v = 1, and returns v.
-func (v *Element) One() *Element {
-	*v = *feOne
-	return v
-}
-
-// reduce reduces v modulo 2^255 - 19 and returns it.
-func (v *Element) reduce() *Element {
-	v.carryPropagate()
-
-	// After the light reduction we now have a field element representation
-	// v < 2^255 + 2^13 * 19, but need v < 2^255 - 19.
-
-	// If v >= 2^255 - 19, then v + 19 >= 2^255, which would overflow 2^255 - 1,
-	// generating a carry. That is, c will be 0 if v < 2^255 - 19, and 1 otherwise.
-	c := (v.l0 + 19) >> 51
-	c = (v.l1 + c) >> 51
-	c = (v.l2 + c) >> 51
-	c = (v.l3 + c) >> 51
-	c = (v.l4 + c) >> 51
-
-	// If v < 2^255 - 19 and c = 0, this will be a no-op. Otherwise, it's
-	// effectively applying the reduction identity to the carry.
-	v.l0 += 19 * c
-
-	v.l1 += v.l0 >> 51
-	v.l0 = v.l0 & maskLow51Bits
-	v.l2 += v.l1 >> 51
-	v.l1 = v.l1 & maskLow51Bits
-	v.l3 += v.l2 >> 51
-	v.l2 = v.l2 & maskLow51Bits
-	v.l4 += v.l3 >> 51
-	v.l3 = v.l3 & maskLow51Bits
-	// no additional carry
-	v.l4 = v.l4 & maskLow51Bits
-
-	return v
-}
-
-// Add sets v = a + b, and returns v.
-func (v *Element) Add(a, b *Element) *Element {
-	v.l0 = a.l0 + b.l0
-	v.l1 = a.l1 + b.l1
-	v.l2 = a.l2 + b.l2
-	v.l3 = a.l3 + b.l3
-	v.l4 = a.l4 + b.l4
-	// Using the generic implementation here is actually faster than the
-	// assembly. Probably because the body of this function is so simple that
-	// the compiler can figure out better optimizations by inlining the carry
-	// propagation. TODO
-	return v.carryPropagateGeneric()
-}
-
-// Subtract sets v = a - b, and returns v.
-func (v *Element) Subtract(a, b *Element) *Element {
-	// We first add 2 * p, to guarantee the subtraction won't underflow, and
-	// then subtract b (which can be up to 2^255 + 2^13 * 19).
-	v.l0 = (a.l0 + 0xFFFFFFFFFFFDA) - b.l0
-	v.l1 = (a.l1 + 0xFFFFFFFFFFFFE) - b.l1
-	v.l2 = (a.l2 + 0xFFFFFFFFFFFFE) - b.l2
-	v.l3 = (a.l3 + 0xFFFFFFFFFFFFE) - b.l3
-	v.l4 = (a.l4 + 0xFFFFFFFFFFFFE) - b.l4
-	return v.carryPropagate()
-}
-
-// Negate sets v = -a, and returns v.
-func (v *Element) Negate(a *Element) *Element {
-	return v.Subtract(feZero, a)
-}
-
-// Invert sets v = 1/z mod p, and returns v.
-//
-// If z == 0, Invert returns v = 0.
-func (v *Element) Invert(z *Element) *Element {
-	// Inversion is implemented as exponentiation with exponent p − 2. It uses the
-	// same sequence of 255 squarings and 11 multiplications as [Curve25519].
-	var z2, z9, z11, z2_5_0, z2_10_0, z2_20_0, z2_50_0, z2_100_0, t Element
-
-	z2.Square(z)             // 2
-	t.Square(&z2)            // 4
-	t.Square(&t)             // 8
-	z9.Multiply(&t, z)       // 9
-	z11.Multiply(&z9, &z2)   // 11
-	t.Square(&z11)           // 22
-	z2_5_0.Multiply(&t, &z9) // 31 = 2^5 - 2^0
-
-	t.Square(&z2_5_0) // 2^6 - 2^1
-	for i := 0; i < 4; i++ {
-		t.Square(&t) // 2^10 - 2^5
-	}
-	z2_10_0.Multiply(&t, &z2_5_0) // 2^10 - 2^0
-
-	t.Square(&z2_10_0) // 2^11 - 2^1
-	for i := 0; i < 9; i++ {
-		t.Square(&t) // 2^20 - 2^10
-	}
-	z2_20_0.Multiply(&t, &z2_10_0) // 2^20 - 2^0
-
-	t.Square(&z2_20_0) // 2^21 - 2^1
-	for i := 0; i < 19; i++ {
-		t.Square(&t) // 2^40 - 2^20
-	}
-	t.Multiply(&t, &z2_20_0) // 2^40 - 2^0
-
-	t.Square(&t) // 2^41 - 2^1
-	for i := 0; i < 9; i++ {
-		t.Square(&t) // 2^50 - 2^10
-	}
-	z2_50_0.Multiply(&t, &z2_10_0) // 2^50 - 2^0
-
-	t.Square(&z2_50_0) // 2^51 - 2^1
-	for i := 0; i < 49; i++ {
-		t.Square(&t) // 2^100 - 2^50
-	}
-	z2_100_0.Multiply(&t, &z2_50_0) // 2^100 - 2^0
-
-	t.Square(&z2_100_0) // 2^101 - 2^1
-	for i := 0; i < 99; i++ {
-		t.Square(&t) // 2^200 - 2^100
-	}
-	t.Multiply(&t, &z2_100_0) // 2^200 - 2^0
-
-	t.Square(&t) // 2^201 - 2^1
-	for i := 0; i < 49; i++ {
-		t.Square(&t) // 2^250 - 2^50
-	}
-	t.Multiply(&t, &z2_50_0) // 2^250 - 2^0
-
-	t.Square(&t) // 2^251 - 2^1
-	t.Square(&t) // 2^252 - 2^2
-	t.Square(&t) // 2^253 - 2^3
-	t.Square(&t) // 2^254 - 2^4
-	t.Square(&t) // 2^255 - 2^5
-
-	return v.Multiply(&t, &z11) // 2^255 - 21
-}
-
-// Set sets v = a, and returns v.
-func (v *Element) Set(a *Element) *Element {
-	*v = *a
-	return v
-}
-
-// SetBytes sets v to x, which must be a 32-byte little-endian encoding.
-//
-// Consistent with RFC 7748, the most significant bit (the high bit of the
-// last byte) is ignored, and non-canonical values (2^255-19 through 2^255-1)
-// are accepted. Note that this is laxer than specified by RFC 8032.
-func (v *Element) SetBytes(x []byte) *Element {
-	if len(x) != 32 {
-		panic("edwards25519: invalid field element input size")
-	}
-
-	// Bits 0:51 (bytes 0:8, bits 0:64, shift 0, mask 51).
-	v.l0 = binary.LittleEndian.Uint64(x[0:8])
-	v.l0 &= maskLow51Bits
-	// Bits 51:102 (bytes 6:14, bits 48:112, shift 3, mask 51).
-	v.l1 = binary.LittleEndian.Uint64(x[6:14]) >> 3
-	v.l1 &= maskLow51Bits
-	// Bits 102:153 (bytes 12:20, bits 96:160, shift 6, mask 51).
-	v.l2 = binary.LittleEndian.Uint64(x[12:20]) >> 6
-	v.l2 &= maskLow51Bits
-	// Bits 153:204 (bytes 19:27, bits 152:216, shift 1, mask 51).
-	v.l3 = binary.LittleEndian.Uint64(x[19:27]) >> 1
-	v.l3 &= maskLow51Bits
-	// Bits 204:251 (bytes 24:32, bits 192:256, shift 12, mask 51).
-	// Note: not bytes 25:33, shift 4, to avoid overread.
-	v.l4 = binary.LittleEndian.Uint64(x[24:32]) >> 12
-	v.l4 &= maskLow51Bits
-
-	return v
-}
-
-// Bytes returns the canonical 32-byte little-endian encoding of v.
-func (v *Element) Bytes() []byte {
-	// This function is outlined to make the allocations inline in the caller
-	// rather than happen on the heap.
-	var out [32]byte
-	return v.bytes(&out)
-}
-
-func (v *Element) bytes(out *[32]byte) []byte {
-	t := *v
-	t.reduce()
-
-	var buf [8]byte
-	for i, l := range [5]uint64{t.l0, t.l1, t.l2, t.l3, t.l4} {
-		bitsOffset := i * 51
-		binary.LittleEndian.PutUint64(buf[:], l<<uint(bitsOffset%8))
-		for i, bb := range buf {
-			off := bitsOffset/8 + i
-			if off >= len(out) {
-				break
-			}
-			out[off] |= bb
-		}
-	}
-
-	return out[:]
-}
-
-// Equal returns 1 if v and u are equal, and 0 otherwise.
-func (v *Element) Equal(u *Element) int {
-	sa, sv := u.Bytes(), v.Bytes()
-	return subtle.ConstantTimeCompare(sa, sv)
-}
-
-// mask64Bits returns 0xffffffff if cond is 1, and 0 otherwise.
-func mask64Bits(cond int) uint64 { return ^(uint64(cond) - 1) }
-
-// Select sets v to a if cond == 1, and to b if cond == 0.
-func (v *Element) Select(a, b *Element, cond int) *Element {
-	m := mask64Bits(cond)
-	v.l0 = (m & a.l0) | (^m & b.l0)
-	v.l1 = (m & a.l1) | (^m & b.l1)
-	v.l2 = (m & a.l2) | (^m & b.l2)
-	v.l3 = (m & a.l3) | (^m & b.l3)
-	v.l4 = (m & a.l4) | (^m & b.l4)
-	return v
-}
-
-// Swap swaps v and u if cond == 1 or leaves them unchanged if cond == 0, and returns v.
-func (v *Element) Swap(u *Element, cond int) {
-	m := mask64Bits(cond)
-	t := m & (v.l0 ^ u.l0)
-	v.l0 ^= t
-	u.l0 ^= t
-	t = m & (v.l1 ^ u.l1)
-	v.l1 ^= t
-	u.l1 ^= t
-	t = m & (v.l2 ^ u.l2)
-	v.l2 ^= t
-	u.l2 ^= t
-	t = m & (v.l3 ^ u.l3)
-	v.l3 ^= t
-	u.l3 ^= t
-	t = m & (v.l4 ^ u.l4)
-	v.l4 ^= t
-	u.l4 ^= t
-}
-
-// IsNegative returns 1 if v is negative, and 0 otherwise.
-func (v *Element) IsNegative() int {
-	return int(v.Bytes()[0] & 1)
-}
-
-// Absolute sets v to |u|, and returns v.
-func (v *Element) Absolute(u *Element) *Element {
-	return v.Select(new(Element).Negate(u), u, u.IsNegative())
-}
-
-// Multiply sets v = x * y, and returns v.
-func (v *Element) Multiply(x, y *Element) *Element {
-	feMul(v, x, y)
-	return v
-}
-
-// Square sets v = x * x, and returns v.
-func (v *Element) Square(x *Element) *Element {
-	feSquare(v, x)
-	return v
-}
-
-// Mult32 sets v = x * y, and returns v.
-func (v *Element) Mult32(x *Element, y uint32) *Element {
-	x0lo, x0hi := mul51(x.l0, y)
-	x1lo, x1hi := mul51(x.l1, y)
-	x2lo, x2hi := mul51(x.l2, y)
-	x3lo, x3hi := mul51(x.l3, y)
-	x4lo, x4hi := mul51(x.l4, y)
-	v.l0 = x0lo + 19*x4hi // carried over per the reduction identity
-	v.l1 = x1lo + x0hi
-	v.l2 = x2lo + x1hi
-	v.l3 = x3lo + x2hi
-	v.l4 = x4lo + x3hi
-	// The hi portions are going to be only 32 bits, plus any previous excess,
-	// so we can skip the carry propagation.
-	return v
-}
-
-// mul51 returns lo + hi * 2⁵¹ = a * b.
-func mul51(a uint64, b uint32) (lo uint64, hi uint64) {
-	mh, ml := bits.Mul64(a, uint64(b))
-	lo = ml & maskLow51Bits
-	hi = (mh << 13) | (ml >> 51)
-	return
-}
-
-// Pow22523 set v = x^((p-5)/8), and returns v. (p-5)/8 is 2^252-3.
-func (v *Element) Pow22523(x *Element) *Element {
-	var t0, t1, t2 Element
-
-	t0.Square(x)             // x^2
-	t1.Square(&t0)           // x^4
-	t1.Square(&t1)           // x^8
-	t1.Multiply(x, &t1)      // x^9
-	t0.Multiply(&t0, &t1)    // x^11
-	t0.Square(&t0)           // x^22
-	t0.Multiply(&t1, &t0)    // x^31
-	t1.Square(&t0)           // x^62
-	for i := 1; i < 5; i++ { // x^992
-		t1.Square(&t1)
-	}
-	t0.Multiply(&t1, &t0)     // x^1023 -> 1023 = 2^10 - 1
-	t1.Square(&t0)            // 2^11 - 2
-	for i := 1; i < 10; i++ { // 2^20 - 2^10
-		t1.Square(&t1)
-	}
-	t1.Multiply(&t1, &t0)     // 2^20 - 1
-	t2.Square(&t1)            // 2^21 - 2
-	for i := 1; i < 20; i++ { // 2^40 - 2^20
-		t2.Square(&t2)
-	}
-	t1.Multiply(&t2, &t1)     // 2^40 - 1
-	t1.Square(&t1)            // 2^41 - 2
-	for i := 1; i < 10; i++ { // 2^50 - 2^10
-		t1.Square(&t1)
-	}
-	t0.Multiply(&t1, &t0)     // 2^50 - 1
-	t1.Square(&t0)            // 2^51 - 2
-	for i := 1; i < 50; i++ { // 2^100 - 2^50
-		t1.Square(&t1)
-	}
-	t1.Multiply(&t1, &t0)      // 2^100 - 1
-	t2.Square(&t1)             // 2^101 - 2
-	for i := 1; i < 100; i++ { // 2^200 - 2^100
-		t2.Square(&t2)
-	}
-	t1.Multiply(&t2, &t1)     // 2^200 - 1
-	t1.Square(&t1)            // 2^201 - 2
-	for i := 1; i < 50; i++ { // 2^250 - 2^50
-		t1.Square(&t1)
-	}
-	t0.Multiply(&t1, &t0)     // 2^250 - 1
-	t0.Square(&t0)            // 2^251 - 2
-	t0.Square(&t0)            // 2^252 - 4
-	return v.Multiply(&t0, x) // 2^252 - 3 -> x^(2^252-3)
-}
-
-// sqrtM1 is 2^((p-1)/4), which squared is equal to -1 by Euler's Criterion.
-var sqrtM1 = &Element{1718705420411056, 234908883556509,
-	2233514472574048, 2117202627021982, 765476049583133}
-
-// SqrtRatio sets r to the non-negative square root of the ratio of u and v.
-//
-// If u/v is square, SqrtRatio returns r and 1. If u/v is not square, SqrtRatio
-// sets r according to Section 4.3 of draft-irtf-cfrg-ristretto255-decaf448-00,
-// and returns r and 0.
-func (r *Element) SqrtRatio(u, v *Element) (rr *Element, wasSquare int) {
-	var a, b Element
-
-	// r = (u * v3) * (u * v7)^((p-5)/8)
-	v2 := a.Square(v)
-	uv3 := b.Multiply(u, b.Multiply(v2, v))
-	uv7 := a.Multiply(uv3, a.Square(v2))
-	r.Multiply(uv3, r.Pow22523(uv7))
-
-	check := a.Multiply(v, a.Square(r)) // check = v * r^2
-
-	uNeg := b.Negate(u)
-	correctSignSqrt := check.Equal(u)
-	flippedSignSqrt := check.Equal(uNeg)
-	flippedSignSqrtI := check.Equal(uNeg.Multiply(uNeg, sqrtM1))
-
-	rPrime := b.Multiply(r, sqrtM1) // r_prime = SQRT_M1 * r
-	// r = CT_SELECT(r_prime IF flipped_sign_sqrt | flipped_sign_sqrt_i ELSE r)
-	r.Select(rPrime, r, flippedSignSqrt|flippedSignSqrtI)
-
-	r.Absolute(r) // Choose the nonnegative square root.
-	return r, correctSignSqrt | flippedSignSqrt
-}

curve25519/internal/field/fe_alias_test.go

@@ -1,126 +0,0 @@
-// Copyright (c) 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package field
-
-import (
-	"testing"
-	"testing/quick"
-)
-
-func checkAliasingOneArg(f func(v, x *Element) *Element) func(v, x Element) bool {
-	return func(v, x Element) bool {
-		x1, v1 := x, x
-
-		// Calculate a reference f(x) without aliasing.
-		if out := f(&v, &x); out != &v && isInBounds(out) {
-			return false
-		}
-
-		// Test aliasing the argument and the receiver.
-		if out := f(&v1, &v1); out != &v1 || v1 != v {
-			return false
-		}
-
-		// Ensure the arguments was not modified.
-		return x == x1
-	}
-}
-
-func checkAliasingTwoArgs(f func(v, x, y *Element) *Element) func(v, x, y Element) bool {
-	return func(v, x, y Element) bool {
-		x1, y1, v1 := x, y, Element{}
-
-		// Calculate a reference f(x, y) without aliasing.
-		if out := f(&v, &x, &y); out != &v && isInBounds(out) {
-			return false
-		}
-
-		// Test aliasing the first argument and the receiver.
-		v1 = x
-		if out := f(&v1, &v1, &y); out != &v1 || v1 != v {
-			return false
-		}
-		// Test aliasing the second argument and the receiver.
-		v1 = y
-		if out := f(&v1, &x, &v1); out != &v1 || v1 != v {
-			return false
-		}
-
-		// Calculate a reference f(x, x) without aliasing.
-		if out := f(&v, &x, &x); out != &v {
-			return false
-		}
-
-		// Test aliasing the first argument and the receiver.
-		v1 = x
-		if out := f(&v1, &v1, &x); out != &v1 || v1 != v {
-			return false
-		}
-		// Test aliasing the second argument and the receiver.
-		v1 = x
-		if out := f(&v1, &x, &v1); out != &v1 || v1 != v {
-			return false
-		}
-		// Test aliasing both arguments and the receiver.
-		v1 = x
-		if out := f(&v1, &v1, &v1); out != &v1 || v1 != v {
-			return false
-		}
-
-		// Ensure the arguments were not modified.
-		return x == x1 && y == y1
-	}
-}
-
-// TestAliasing checks that receivers and arguments can alias each other without
-// leading to incorrect results. That is, it ensures that it's safe to write
-//
-//	v.Invert(v)
-//
-// or
-//
-//	v.Add(v, v)
-//
-// without any of the inputs getting clobbered by the output being written.
-func TestAliasing(t *testing.T) {
-	type target struct {
-		name     string
-		oneArgF  func(v, x *Element) *Element
-		twoArgsF func(v, x, y *Element) *Element
-	}
-	for _, tt := range []target{
-		{name: "Absolute", oneArgF: (*Element).Absolute},
-		{name: "Invert", oneArgF: (*Element).Invert},
-		{name: "Negate", oneArgF: (*Element).Negate},
-		{name: "Set", oneArgF: (*Element).Set},
-		{name: "Square", oneArgF: (*Element).Square},
-		{name: "Multiply", twoArgsF: (*Element).Multiply},
-		{name: "Add", twoArgsF: (*Element).Add},
-		{name: "Subtract", twoArgsF: (*Element).Subtract},
-		{
-			name: "Select0",
-			twoArgsF: func(v, x, y *Element) *Element {
-				return (*Element).Select(v, x, y, 0)
-			},
-		},
-		{
-			name: "Select1",
-			twoArgsF: func(v, x, y *Element) *Element {
-				return (*Element).Select(v, x, y, 1)
-			},
-		},
-	} {
-		var err error
-		switch {
-		case tt.oneArgF != nil:
-			err = quick.Check(checkAliasingOneArg(tt.oneArgF), &quick.Config{MaxCountScale: 1 << 8})
-		case tt.twoArgsF != nil:
-			err = quick.Check(checkAliasingTwoArgs(tt.twoArgsF), &quick.Config{MaxCountScale: 1 << 8})
-		}
-		if err != nil {
-			t.Errorf("%v: %v", tt.name, err)
-		}
-	}
-}

curve25519/internal/field/fe_amd64.go

@@ -1,15 +0,0 @@
-// Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT.
-
-//go:build amd64 && gc && !purego
-
-package field
-
-// feMul sets out = a * b. It works like feMulGeneric.
-//
-//go:noescape
-func feMul(out *Element, a *Element, b *Element)
-
-// feSquare sets out = a * a. It works like feSquareGeneric.
-//
-//go:noescape
-func feSquare(out *Element, a *Element)

curve25519/internal/field/fe_amd64.s

@@ -1,378 +0,0 @@
-// Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT.
-
-//go:build amd64 && gc && !purego
-
-#include "textflag.h"
-
-// func feMul(out *Element, a *Element, b *Element)
-TEXT ·feMul(SB), NOSPLIT, $0-24
-	MOVQ a+8(FP), CX
-	MOVQ b+16(FP), BX
-
-	// r0 = a0×b0
-	MOVQ (CX), AX
-	MULQ (BX)
-	MOVQ AX, DI
-	MOVQ DX, SI
-
-	// r0 += 19×a1×b4
-	MOVQ   8(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   32(BX)
-	ADDQ   AX, DI
-	ADCQ   DX, SI
-
-	// r0 += 19×a2×b3
-	MOVQ   16(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   24(BX)
-	ADDQ   AX, DI
-	ADCQ   DX, SI
-
-	// r0 += 19×a3×b2
-	MOVQ   24(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   16(BX)
-	ADDQ   AX, DI
-	ADCQ   DX, SI
-
-	// r0 += 19×a4×b1
-	MOVQ   32(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   8(BX)
-	ADDQ   AX, DI
-	ADCQ   DX, SI
-
-	// r1 = a0×b1
-	MOVQ (CX), AX
-	MULQ 8(BX)
-	MOVQ AX, R9
-	MOVQ DX, R8
-
-	// r1 += a1×b0
-	MOVQ 8(CX), AX
-	MULQ (BX)
-	ADDQ AX, R9
-	ADCQ DX, R8
-
-	// r1 += 19×a2×b4
-	MOVQ   16(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   32(BX)
-	ADDQ   AX, R9
-	ADCQ   DX, R8
-
-	// r1 += 19×a3×b3
-	MOVQ   24(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   24(BX)
-	ADDQ   AX, R9
-	ADCQ   DX, R8
-
-	// r1 += 19×a4×b2
-	MOVQ   32(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   16(BX)
-	ADDQ   AX, R9
-	ADCQ   DX, R8
-
-	// r2 = a0×b2
-	MOVQ (CX), AX
-	MULQ 16(BX)
-	MOVQ AX, R11
-	MOVQ DX, R10
-
-	// r2 += a1×b1
-	MOVQ 8(CX), AX
-	MULQ 8(BX)
-	ADDQ AX, R11
-	ADCQ DX, R10
-
-	// r2 += a2×b0
-	MOVQ 16(CX), AX
-	MULQ (BX)
-	ADDQ AX, R11
-	ADCQ DX, R10
-
-	// r2 += 19×a3×b4
-	MOVQ   24(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   32(BX)
-	ADDQ   AX, R11
-	ADCQ   DX, R10
-
-	// r2 += 19×a4×b3
-	MOVQ   32(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   24(BX)
-	ADDQ   AX, R11
-	ADCQ   DX, R10
-
-	// r3 = a0×b3
-	MOVQ (CX), AX
-	MULQ 24(BX)
-	MOVQ AX, R13
-	MOVQ DX, R12
-
-	// r3 += a1×b2
-	MOVQ 8(CX), AX
-	MULQ 16(BX)
-	ADDQ AX, R13
-	ADCQ DX, R12
-
-	// r3 += a2×b1
-	MOVQ 16(CX), AX
-	MULQ 8(BX)
-	ADDQ AX, R13
-	ADCQ DX, R12
-
-	// r3 += a3×b0
-	MOVQ 24(CX), AX
-	MULQ (BX)
-	ADDQ AX, R13
-	ADCQ DX, R12
-
-	// r3 += 19×a4×b4
-	MOVQ   32(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   32(BX)
-	ADDQ   AX, R13
-	ADCQ   DX, R12
-
-	// r4 = a0×b4
-	MOVQ (CX), AX
-	MULQ 32(BX)
-	MOVQ AX, R15
-	MOVQ DX, R14
-
-	// r4 += a1×b3
-	MOVQ 8(CX), AX
-	MULQ 24(BX)
-	ADDQ AX, R15
-	ADCQ DX, R14
-
-	// r4 += a2×b2
-	MOVQ 16(CX), AX
-	MULQ 16(BX)
-	ADDQ AX, R15
-	ADCQ DX, R14
-
-	// r4 += a3×b1
-	MOVQ 24(CX), AX
-	MULQ 8(BX)
-	ADDQ AX, R15
-	ADCQ DX, R14
-
-	// r4 += a4×b0
-	MOVQ 32(CX), AX
-	MULQ (BX)
-	ADDQ AX, R15
-	ADCQ DX, R14
-
-	// First reduction chain
-	MOVQ   $0x0007ffffffffffff, AX
-	SHLQ   $0x0d, DI, SI
-	SHLQ   $0x0d, R9, R8
-	SHLQ   $0x0d, R11, R10
-	SHLQ   $0x0d, R13, R12
-	SHLQ   $0x0d, R15, R14
-	ANDQ   AX, DI
-	IMUL3Q $0x13, R14, R14
-	ADDQ   R14, DI
-	ANDQ   AX, R9
-	ADDQ   SI, R9
-	ANDQ   AX, R11
-	ADDQ   R8, R11
-	ANDQ   AX, R13
-	ADDQ   R10, R13
-	ANDQ   AX, R15
-	ADDQ   R12, R15
-
-	// Second reduction chain (carryPropagate)
-	MOVQ   DI, SI
-	SHRQ   $0x33, SI
-	MOVQ   R9, R8
-	SHRQ   $0x33, R8
-	MOVQ   R11, R10
-	SHRQ   $0x33, R10
-	MOVQ   R13, R12
-	SHRQ   $0x33, R12
-	MOVQ   R15, R14
-	SHRQ   $0x33, R14
-	ANDQ   AX, DI
-	IMUL3Q $0x13, R14, R14
-	ADDQ   R14, DI
-	ANDQ   AX, R9
-	ADDQ   SI, R9
-	ANDQ   AX, R11
-	ADDQ   R8, R11
-	ANDQ   AX, R13
-	ADDQ   R10, R13
-	ANDQ   AX, R15
-	ADDQ   R12, R15
-
-	// Store output
-	MOVQ out+0(FP), AX
-	MOVQ DI, (AX)
-	MOVQ R9, 8(AX)
-	MOVQ R11, 16(AX)
-	MOVQ R13, 24(AX)
-	MOVQ R15, 32(AX)
-	RET
-
-// func feSquare(out *Element, a *Element)
-TEXT ·feSquare(SB), NOSPLIT, $0-16
-	MOVQ a+8(FP), CX
-
-	// r0 = l0×l0
-	MOVQ (CX), AX
-	MULQ (CX)
-	MOVQ AX, SI
-	MOVQ DX, BX
-
-	// r0 += 38×l1×l4
-	MOVQ   8(CX), AX
-	IMUL3Q $0x26, AX, AX
-	MULQ   32(CX)
-	ADDQ   AX, SI
-	ADCQ   DX, BX
-
-	// r0 += 38×l2×l3
-	MOVQ   16(CX), AX
-	IMUL3Q $0x26, AX, AX
-	MULQ   24(CX)
-	ADDQ   AX, SI
-	ADCQ   DX, BX
-
-	// r1 = 2×l0×l1
-	MOVQ (CX), AX
-	SHLQ $0x01, AX
-	MULQ 8(CX)
-	MOVQ AX, R8
-	MOVQ DX, DI
-
-	// r1 += 38×l2×l4
-	MOVQ   16(CX), AX
-	IMUL3Q $0x26, AX, AX
-	MULQ   32(CX)
-	ADDQ   AX, R8
-	ADCQ   DX, DI
-
-	// r1 += 19×l3×l3
-	MOVQ   24(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   24(CX)
-	ADDQ   AX, R8
-	ADCQ   DX, DI
-
-	// r2 = 2×l0×l2
-	MOVQ (CX), AX
-	SHLQ $0x01, AX
-	MULQ 16(CX)
-	MOVQ AX, R10
-	MOVQ DX, R9
-
-	// r2 += l1×l1
-	MOVQ 8(CX), AX
-	MULQ 8(CX)
-	ADDQ AX, R10
-	ADCQ DX, R9
-
-	// r2 += 38×l3×l4
-	MOVQ   24(CX), AX
-	IMUL3Q $0x26, AX, AX
-	MULQ   32(CX)
-	ADDQ   AX, R10
-	ADCQ   DX, R9
-
-	// r3 = 2×l0×l3
-	MOVQ (CX), AX
-	SHLQ $0x01, AX
-	MULQ 24(CX)
-	MOVQ AX, R12
-	MOVQ DX, R11
-
-	// r3 += 2×l1×l2
-	MOVQ   8(CX), AX
-	IMUL3Q $0x02, AX, AX
-	MULQ   16(CX)
-	ADDQ   AX, R12
-	ADCQ   DX, R11
-
-	// r3 += 19×l4×l4
-	MOVQ   32(CX), AX
-	IMUL3Q $0x13, AX, AX
-	MULQ   32(CX)
-	ADDQ   AX, R12
-	ADCQ   DX, R11
-
-	// r4 = 2×l0×l4
-	MOVQ (CX), AX
-	SHLQ $0x01, AX
-	MULQ 32(CX)
-	MOVQ AX, R14
-	MOVQ DX, R13
-
-	// r4 += 2×l1×l3
-	MOVQ   8(CX), AX
-	IMUL3Q $0x02, AX, AX
-	MULQ   24(CX)
-	ADDQ   AX, R14
-	ADCQ   DX, R13
-
-	// r4 += l2×l2
-	MOVQ 16(CX), AX
-	MULQ 16(CX)
-	ADDQ AX, R14
-	ADCQ DX, R13
-
-	// First reduction chain
-	MOVQ   $0x0007ffffffffffff, AX
-	SHLQ   $0x0d, SI, BX
-	SHLQ   $0x0d, R8, DI
-	SHLQ   $0x0d, R10, R9
-	SHLQ   $0x0d, R12, R11
-	SHLQ   $0x0d, R14, R13
-	ANDQ   AX, SI
-	IMUL3Q $0x13, R13, R13
-	ADDQ   R13, SI
-	ANDQ   AX, R8
-	ADDQ   BX, R8
-	ANDQ   AX, R10
-	ADDQ   DI, R10
-	ANDQ   AX, R12
-	ADDQ   R9, R12
-	ANDQ   AX, R14
-	ADDQ   R11, R14
-
-	// Second reduction chain (carryPropagate)
-	MOVQ   SI, BX
-	SHRQ   $0x33, BX
-	MOVQ   R8, DI
-	SHRQ   $0x33, DI
-	MOVQ   R10, R9
-	SHRQ   $0x33, R9
-	MOVQ   R12, R11
-	SHRQ   $0x33, R11
-	MOVQ   R14, R13
-	SHRQ   $0x33, R13
-	ANDQ   AX, SI
-	IMUL3Q $0x13, R13, R13
-	ADDQ   R13, SI
-	ANDQ   AX, R8
-	ADDQ   BX, R8
-	ANDQ   AX, R10
-	ADDQ   DI, R10
-	ANDQ   AX, R12
-	ADDQ   R9, R12
-	ANDQ   AX, R14
-	ADDQ   R11, R14
-
-	// Store output
-	MOVQ out+0(FP), AX
-	MOVQ SI, (AX)
-	MOVQ R8, 8(AX)
-	MOVQ R10, 16(AX)
-	MOVQ R12, 24(AX)
-	MOVQ R14, 32(AX)
-	RET

curve25519/internal/field/fe_amd64_noasm.go

@@ -1,11 +0,0 @@
-// Copyright (c) 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build !amd64 || !gc || purego
-
-package field
-
-func feMul(v, x, y *Element) { feMulGeneric(v, x, y) }
-
-func feSquare(v, x *Element) { feSquareGeneric(v, x) }

curve25519/internal/field/fe_arm64.go

@@ -1,15 +0,0 @@
-// Copyright (c) 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build arm64 && gc && !purego
-
-package field
-
-//go:noescape
-func carryPropagate(v *Element)
-
-func (v *Element) carryPropagate() *Element {
-	carryPropagate(v)
-	return v
-}

curve25519/internal/field/fe_arm64.s

@@ -1,42 +0,0 @@
-// Copyright (c) 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build arm64 && gc && !purego
-
-#include "textflag.h"
-
-// carryPropagate works exactly like carryPropagateGeneric and uses the
-// same AND, ADD, and LSR+MADD instructions emitted by the compiler, but
-// avoids loading R0-R4 twice and uses LDP and STP.
-//
-// See https://golang.org/issues/43145 for the main compiler issue.
-//
-// func carryPropagate(v *Element)
-TEXT ·carryPropagate(SB),NOFRAME|NOSPLIT,$0-8
-	MOVD v+0(FP), R20
-
-	LDP 0(R20), (R0, R1)
-	LDP 16(R20), (R2, R3)
-	MOVD 32(R20), R4
-
-	AND $0x7ffffffffffff, R0, R10
-	AND $0x7ffffffffffff, R1, R11
-	AND $0x7ffffffffffff, R2, R12
-	AND $0x7ffffffffffff, R3, R13
-	AND $0x7ffffffffffff, R4, R14
-
-	ADD R0>>51, R11, R11
-	ADD R1>>51, R12, R12
-	ADD R2>>51, R13, R13
-	ADD R3>>51, R14, R14
-	// R4>>51 * 19 + R10 -> R10
-	LSR $51, R4, R21
-	MOVD $19, R22
-	MADD R22, R10, R21, R10
-
-	STP (R10, R11), 0(R20)
-	STP (R12, R13), 16(R20)
-	MOVD R14, 32(R20)
-
-	RET

curve25519/internal/field/fe_arm64_noasm.go

@@ -1,11 +0,0 @@
-// Copyright (c) 2021 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build !arm64 || !gc || purego
-
-package field
-
-func (v *Element) carryPropagate() *Element {
-	return v.carryPropagateGeneric()
-}

curve25519/internal/field/fe_bench_test.go

@@ -1,36 +0,0 @@
-// Copyright (c) 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package field
-
-import "testing"
-
-func BenchmarkAdd(b *testing.B) {
-	var x, y Element
-	x.One()
-	y.Add(feOne, feOne)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		x.Add(&x, &y)
-	}
-}
-
-func BenchmarkMultiply(b *testing.B) {
-	var x, y Element
-	x.One()
-	y.Add(feOne, feOne)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		x.Multiply(&x, &y)
-	}
-}
-
-func BenchmarkMult32(b *testing.B) {
-	var x Element
-	x.One()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		x.Mult32(&x, 0xaa42aa42)
-	}
-}

curve25519/internal/field/fe_generic.go

@@ -1,264 +0,0 @@
-// Copyright (c) 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package field
-
-import "math/bits"
-
-// uint128 holds a 128-bit number as two 64-bit limbs, for use with the
-// bits.Mul64 and bits.Add64 intrinsics.
-type uint128 struct {
-	lo, hi uint64
-}
-
-// mul64 returns a * b.
-func mul64(a, b uint64) uint128 {
-	hi, lo := bits.Mul64(a, b)
-	return uint128{lo, hi}
-}
-
-// addMul64 returns v + a * b.
-func addMul64(v uint128, a, b uint64) uint128 {
-	hi, lo := bits.Mul64(a, b)
-	lo, c := bits.Add64(lo, v.lo, 0)
-	hi, _ = bits.Add64(hi, v.hi, c)
-	return uint128{lo, hi}
-}
-
-// shiftRightBy51 returns a >> 51. a is assumed to be at most 115 bits.
-func shiftRightBy51(a uint128) uint64 {
-	return (a.hi << (64 - 51)) | (a.lo >> 51)
-}
-
-func feMulGeneric(v, a, b *Element) {
-	a0 := a.l0
-	a1 := a.l1
-	a2 := a.l2
-	a3 := a.l3
-	a4 := a.l4
-
-	b0 := b.l0
-	b1 := b.l1
-	b2 := b.l2
-	b3 := b.l3
-	b4 := b.l4
-
-	// Limb multiplication works like pen-and-paper columnar multiplication, but
-	// with 51-bit limbs instead of digits.
-	//
-	//                          a4   a3   a2   a1   a0  x
-	//                          b4   b3   b2   b1   b0  =
-	//                         ------------------------
-	//                        a4b0 a3b0 a2b0 a1b0 a0b0  +
-	//                   a4b1 a3b1 a2b1 a1b1 a0b1       +
-	//              a4b2 a3b2 a2b2 a1b2 a0b2            +
-	//         a4b3 a3b3 a2b3 a1b3 a0b3                 +
-	//    a4b4 a3b4 a2b4 a1b4 a0b4                      =
-	//   ----------------------------------------------
-	//      r8   r7   r6   r5   r4   r3   r2   r1   r0
-	//
-	// We can then use the reduction identity (a * 2²⁵⁵ + b = a * 19 + b) to
-	// reduce the limbs that would overflow 255 bits. r5 * 2²⁵⁵ becomes 19 * r5,
-	// r6 * 2³⁰⁶ becomes 19 * r6 * 2⁵¹, etc.
-	//
-	// Reduction can be carried out simultaneously to multiplication. For
-	// example, we do not compute r5: whenever the result of a multiplication
-	// belongs to r5, like a1b4, we multiply it by 19 and add the result to r0.
-	//
-	//            a4b0    a3b0    a2b0    a1b0    a0b0  +
-	//            a3b1    a2b1    a1b1    a0b1 19×a4b1  +
-	//            a2b2    a1b2    a0b2 19×a4b2 19×a3b2  +
-	//            a1b3    a0b3 19×a4b3 19×a3b3 19×a2b3  +
-	//            a0b4 19×a4b4 19×a3b4 19×a2b4 19×a1b4  =
-	//           --------------------------------------
-	//              r4      r3      r2      r1      r0
-	//
-	// Finally we add up the columns into wide, overlapping limbs.
-
-	a1_19 := a1 * 19
-	a2_19 := a2 * 19
-	a3_19 := a3 * 19
-	a4_19 := a4 * 19
-
-	// r0 = a0×b0 + 19×(a1×b4 + a2×b3 + a3×b2 + a4×b1)
-	r0 := mul64(a0, b0)
-	r0 = addMul64(r0, a1_19, b4)
-	r0 = addMul64(r0, a2_19, b3)
-	r0 = addMul64(r0, a3_19, b2)
-	r0 = addMul64(r0, a4_19, b1)
-
-	// r1 = a0×b1 + a1×b0 + 19×(a2×b4 + a3×b3 + a4×b2)
-	r1 := mul64(a0, b1)
-	r1 = addMul64(r1, a1, b0)
-	r1 = addMul64(r1, a2_19, b4)
-	r1 = addMul64(r1, a3_19, b3)
-	r1 = addMul64(r1, a4_19, b2)
-
-	// r2 = a0×b2 + a1×b1 + a2×b0 + 19×(a3×b4 + a4×b3)
-	r2 := mul64(a0, b2)
-	r2 = addMul64(r2, a1, b1)
-	r2 = addMul64(r2, a2, b0)
-	r2 = addMul64(r2, a3_19, b4)
-	r2 = addMul64(r2, a4_19, b3)
-
-	// r3 = a0×b3 + a1×b2 + a2×b1 + a3×b0 + 19×a4×b4
-	r3 := mul64(a0, b3)
-	r3 = addMul64(r3, a1, b2)
-	r3 = addMul64(r3, a2, b1)
-	r3 = addMul64(r3, a3, b0)
-	r3 = addMul64(r3, a4_19, b4)
-
-	// r4 = a0×b4 + a1×b3 + a2×b2 + a3×b1 + a4×b0
-	r4 := mul64(a0, b4)
-	r4 = addMul64(r4, a1, b3)
-	r4 = addMul64(r4, a2, b2)
-	r4 = addMul64(r4, a3, b1)
-	r4 = addMul64(r4, a4, b0)
-
-	// After the multiplication, we need to reduce (carry) the five coefficients
-	// to obtain a result with limbs that are at most slightly larger than 2⁵¹,
-	// to respect the Element invariant.
-	//
-	// Overall, the reduction works the same as carryPropagate, except with
-	// wider inputs: we take the carry for each coefficient by shifting it right
-	// by 51, and add it to the limb above it. The top carry is multiplied by 19
-	// according to the reduction identity and added to the lowest limb.
-	//
-	// The largest coefficient (r0) will be at most 111 bits, which guarantees
-	// that all carries are at most 111 - 51 = 60 bits, which fits in a uint64.
-	//
-	//     r0 = a0×b0 + 19×(a1×b4 + a2×b3 + a3×b2 + a4×b1)
-	//     r0 < 2⁵²×2⁵² + 19×(2⁵²×2⁵² + 2⁵²×2⁵² + 2⁵²×2⁵² + 2⁵²×2⁵²)
-	//     r0 < (1 + 19 × 4) × 2⁵² × 2⁵²
-	//     r0 < 2⁷ × 2⁵² × 2⁵²
-	//     r0 < 2¹¹¹
-	//
-	// Moreover, the top coefficient (r4) is at most 107 bits, so c4 is at most
-	// 56 bits, and c4 * 19 is at most 61 bits, which again fits in a uint64 and
-	// allows us to easily apply the reduction identity.
-	//
-	//     r4 = a0×b4 + a1×b3 + a2×b2 + a3×b1 + a4×b0
-	//     r4 < 5 × 2⁵² × 2⁵²
-	//     r4 < 2¹⁰⁷
-	//
-
-	c0 := shiftRightBy51(r0)
-	c1 := shiftRightBy51(r1)
-	c2 := shiftRightBy51(r2)
-	c3 := shiftRightBy51(r3)
-	c4 := shiftRightBy51(r4)
-
-	rr0 := r0.lo&maskLow51Bits + c4*19
-	rr1 := r1.lo&maskLow51Bits + c0
-	rr2 := r2.lo&maskLow51Bits + c1
-	rr3 := r3.lo&maskLow51Bits + c2
-	rr4 := r4.lo&maskLow51Bits + c3
-
-	// Now all coefficients fit into 64-bit registers but are still too large to
-	// be passed around as a Element. We therefore do one last carry chain,
-	// where the carries will be small enough to fit in the wiggle room above 2⁵¹.
-	*v = Element{rr0, rr1, rr2, rr3, rr4}
-	v.carryPropagate()
-}
-
-func feSquareGeneric(v, a *Element) {
-	l0 := a.l0
-	l1 := a.l1
-	l2 := a.l2
-	l3 := a.l3
-	l4 := a.l4
-
-	// Squaring works precisely like multiplication above, but thanks to its
-	// symmetry we get to group a few terms together.
-	//
-	//                          l4   l3   l2   l1   l0  x
-	//                          l4   l3   l2   l1   l0  =
-	//                         ------------------------
-	//                        l4l0 l3l0 l2l0 l1l0 l0l0  +
-	//                   l4l1 l3l1 l2l1 l1l1 l0l1       +
-	//              l4l2 l3l2 l2l2 l1l2 l0l2            +
-	//         l4l3 l3l3 l2l3 l1l3 l0l3                 +
-	//    l4l4 l3l4 l2l4 l1l4 l0l4                      =
-	//   ----------------------------------------------
-	//      r8   r7   r6   r5   r4   r3   r2   r1   r0
-	//
-	//            l4l0    l3l0    l2l0    l1l0    l0l0  +
-	//            l3l1    l2l1    l1l1    l0l1 19×l4l1  +
-	//            l2l2    l1l2    l0l2 19×l4l2 19×l3l2  +
-	//            l1l3    l0l3 19×l4l3 19×l3l3 19×l2l3  +
-	//            l0l4 19×l4l4 19×l3l4 19×l2l4 19×l1l4  =
-	//           --------------------------------------
-	//              r4      r3      r2      r1      r0
-	//
-	// With precomputed 2×, 19×, and 2×19× terms, we can compute each limb with
-	// only three Mul64 and four Add64, instead of five and eight.
-
-	l0_2 := l0 * 2
-	l1_2 := l1 * 2
-
-	l1_38 := l1 * 38
-	l2_38 := l2 * 38
-	l3_38 := l3 * 38
-
-	l3_19 := l3 * 19
-	l4_19 := l4 * 19
-
-	// r0 = l0×l0 + 19×(l1×l4 + l2×l3 + l3×l2 + l4×l1) = l0×l0 + 19×2×(l1×l4 + l2×l3)
-	r0 := mul64(l0, l0)
-	r0 = addMul64(r0, l1_38, l4)
-	r0 = addMul64(r0, l2_38, l3)
-
-	// r1 = l0×l1 + l1×l0 + 19×(l2×l4 + l3×l3 + l4×l2) = 2×l0×l1 + 19×2×l2×l4 + 19×l3×l3
-	r1 := mul64(l0_2, l1)
-	r1 = addMul64(r1, l2_38, l4)
-	r1 = addMul64(r1, l3_19, l3)
-
-	// r2 = l0×l2 + l1×l1 + l2×l0 + 19×(l3×l4 + l4×l3) = 2×l0×l2 + l1×l1 + 19×2×l3×l4
-	r2 := mul64(l0_2, l2)
-	r2 = addMul64(r2, l1, l1)
-	r2 = addMul64(r2, l3_38, l4)
-
-	// r3 = l0×l3 + l1×l2 + l2×l1 + l3×l0 + 19×l4×l4 = 2×l0×l3 + 2×l1×l2 + 19×l4×l4
-	r3 := mul64(l0_2, l3)
-	r3 = addMul64(r3, l1_2, l2)
-	r3 = addMul64(r3, l4_19, l4)
-
-	// r4 = l0×l4 + l1×l3 + l2×l2 + l3×l1 + l4×l0 = 2×l0×l4 + 2×l1×l3 + l2×l2
-	r4 := mul64(l0_2, l4)
-	r4 = addMul64(r4, l1_2, l3)
-	r4 = addMul64(r4, l2, l2)
-
-	c0 := shiftRightBy51(r0)
-	c1 := shiftRightBy51(r1)
-	c2 := shiftRightBy51(r2)
-	c3 := shiftRightBy51(r3)
-	c4 := shiftRightBy51(r4)
-
-	rr0 := r0.lo&maskLow51Bits + c4*19
-	rr1 := r1.lo&maskLow51Bits + c0
-	rr2 := r2.lo&maskLow51Bits + c1
-	rr3 := r3.lo&maskLow51Bits + c2
-	rr4 := r4.lo&maskLow51Bits + c3
-
-	*v = Element{rr0, rr1, rr2, rr3, rr4}
-	v.carryPropagate()
-}
-
-// carryPropagateGeneric brings the limbs below 52 bits by applying the reduction
-// identity (a * 2²⁵⁵ + b = a * 19 + b) to the l4 carry. TODO inline
-func (v *Element) carryPropagateGeneric() *Element {
-	c0 := v.l0 >> 51
-	c1 := v.l1 >> 51
-	c2 := v.l2 >> 51
-	c3 := v.l3 >> 51
-	c4 := v.l4 >> 51
-
-	v.l0 = v.l0&maskLow51Bits + c4*19
-	v.l1 = v.l1&maskLow51Bits + c0
-	v.l2 = v.l2&maskLow51Bits + c1
-	v.l3 = v.l3&maskLow51Bits + c2
-	v.l4 = v.l4&maskLow51Bits + c3
-
-	return v
-}

curve25519/internal/field/fe_test.go

@@ -1,558 +0,0 @@
-// Copyright (c) 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package field
-
-import (
-	"bytes"
-	"crypto/rand"
-	"encoding/hex"
-	"io"
-	"math/big"
-	"math/bits"
-	mathrand "math/rand"
-	"reflect"
-	"testing"
-	"testing/quick"
-)
-
-func (v Element) String() string {
-	return hex.EncodeToString(v.Bytes())
-}
-
-// quickCheckConfig1024 will make each quickcheck test run (1024 * -quickchecks)
-// times. The default value of -quickchecks is 100.
-var quickCheckConfig1024 = &quick.Config{MaxCountScale: 1 << 10}
-
-func generateFieldElement(rand *mathrand.Rand) Element {
-	const maskLow52Bits = (1 << 52) - 1
-	return Element{
-		rand.Uint64() & maskLow52Bits,
-		rand.Uint64() & maskLow52Bits,
-		rand.Uint64() & maskLow52Bits,
-		rand.Uint64() & maskLow52Bits,
-		rand.Uint64() & maskLow52Bits,
-	}
-}
-
-// weirdLimbs can be combined to generate a range of edge-case field elements.
-// 0 and -1 are intentionally more weighted, as they combine well.
-var (
-	weirdLimbs51 = []uint64{
-		0, 0, 0, 0,
-		1,
-		19 - 1,
-		19,
-		0x2aaaaaaaaaaaa,
-		0x5555555555555,
-		(1 << 51) - 20,
-		(1 << 51) - 19,
-		(1 << 51) - 1, (1 << 51) - 1,
-		(1 << 51) - 1, (1 << 51) - 1,
-	}
-	weirdLimbs52 = []uint64{
-		0, 0, 0, 0, 0, 0,
-		1,
-		19 - 1,
-		19,
-		0x2aaaaaaaaaaaa,
-		0x5555555555555,
-		(1 << 51) - 20,
-		(1 << 51) - 19,
-		(1 << 51) - 1, (1 << 51) - 1,
-		(1 << 51) - 1, (1 << 51) - 1,
-		(1 << 51) - 1, (1 << 51) - 1,
-		1 << 51,
-		(1 << 51) + 1,
-		(1 << 52) - 19,
-		(1 << 52) - 1,
-	}
-)
-
-func generateWeirdFieldElement(rand *mathrand.Rand) Element {
-	return Element{
-		weirdLimbs52[rand.Intn(len(weirdLimbs52))],
-		weirdLimbs51[rand.Intn(len(weirdLimbs51))],
-		weirdLimbs51[rand.Intn(len(weirdLimbs51))],
-		weirdLimbs51[rand.Intn(len(weirdLimbs51))],
-		weirdLimbs51[rand.Intn(len(weirdLimbs51))],
-	}
-}
-
-func (Element) Generate(rand *mathrand.Rand, size int) reflect.Value {
-	if rand.Intn(2) == 0 {
-		return reflect.ValueOf(generateWeirdFieldElement(rand))
-	}
-	return reflect.ValueOf(generateFieldElement(rand))
-}
-
-// isInBounds returns whether the element is within the expected bit size bounds
-// after a light reduction.
-func isInBounds(x *Element) bool {
-	return bits.Len64(x.l0) <= 52 &&
-		bits.Len64(x.l1) <= 52 &&
-		bits.Len64(x.l2) <= 52 &&
-		bits.Len64(x.l3) <= 52 &&
-		bits.Len64(x.l4) <= 52
-}
-
-func TestMultiplyDistributesOverAdd(t *testing.T) {
-	multiplyDistributesOverAdd := func(x, y, z Element) bool {
-		// Compute t1 = (x+y)*z
-		t1 := new(Element)
-		t1.Add(&x, &y)
-		t1.Multiply(t1, &z)
-
-		// Compute t2 = x*z + y*z
-		t2 := new(Element)
-		t3 := new(Element)
-		t2.Multiply(&x, &z)
-		t3.Multiply(&y, &z)
-		t2.Add(t2, t3)
-
-		return t1.Equal(t2) == 1 && isInBounds(t1) && isInBounds(t2)
-	}
-
-	if err := quick.Check(multiplyDistributesOverAdd, quickCheckConfig1024); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestMul64to128(t *testing.T) {
-	a := uint64(5)
-	b := uint64(5)
-	r := mul64(a, b)
-	if r.lo != 0x19 || r.hi != 0 {
-		t.Errorf("lo-range wide mult failed, got %d + %d*(2**64)", r.lo, r.hi)
-	}
-
-	a = uint64(18014398509481983) // 2^54 - 1
-	b = uint64(18014398509481983) // 2^54 - 1
-	r = mul64(a, b)
-	if r.lo != 0xff80000000000001 || r.hi != 0xfffffffffff {
-		t.Errorf("hi-range wide mult failed, got %d + %d*(2**64)", r.lo, r.hi)
-	}
-
-	a = uint64(1125899906842661)
-	b = uint64(2097155)
-	r = mul64(a, b)
-	r = addMul64(r, a, b)
-	r = addMul64(r, a, b)
-	r = addMul64(r, a, b)
-	r = addMul64(r, a, b)
-	if r.lo != 16888498990613035 || r.hi != 640 {
-		t.Errorf("wrong answer: %d + %d*(2**64)", r.lo, r.hi)
-	}
-}
-
-func TestSetBytesRoundTrip(t *testing.T) {
-	f1 := func(in [32]byte, fe Element) bool {
-		fe.SetBytes(in[:])
-
-		// Mask the most significant bit as it's ignored by SetBytes. (Now
-		// instead of earlier so we check the masking in SetBytes is working.)
-		in[len(in)-1] &= (1 << 7) - 1
-
-		return bytes.Equal(in[:], fe.Bytes()) && isInBounds(&fe)
-	}
-	if err := quick.Check(f1, nil); err != nil {
-		t.Errorf("failed bytes->FE->bytes round-trip: %v", err)
-	}
-
-	f2 := func(fe, r Element) bool {
-		r.SetBytes(fe.Bytes())
-
-		// Intentionally not using Equal not to go through Bytes again.
-		// Calling reduce because both Generate and SetBytes can produce
-		// non-canonical representations.
-		fe.reduce()
-		r.reduce()
-		return fe == r
-	}
-	if err := quick.Check(f2, nil); err != nil {
-		t.Errorf("failed FE->bytes->FE round-trip: %v", err)
-	}
-
-	// Check some fixed vectors from dalek
-	type feRTTest struct {
-		fe Element
-		b  []byte
-	}
-	var tests = []feRTTest{
-		{
-			fe: Element{358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676},
-			b:  []byte{74, 209, 69, 197, 70, 70, 161, 222, 56, 226, 229, 19, 112, 60, 25, 92, 187, 74, 222, 56, 50, 153, 51, 233, 40, 74, 57, 6, 160, 185, 213, 31},
-		},
-		{
-			fe: Element{84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972},
-			b:  []byte{199, 23, 106, 112, 61, 77, 216, 79, 186, 60, 11, 118, 13, 16, 103, 15, 42, 32, 83, 250, 44, 57, 204, 198, 78, 199, 253, 119, 146, 172, 3, 122},
-		},
-	}
-
-	for _, tt := range tests {
-		b := tt.fe.Bytes()
-		if !bytes.Equal(b, tt.b) || new(Element).SetBytes(tt.b).Equal(&tt.fe) != 1 {
-			t.Errorf("Failed fixed roundtrip: %v", tt)
-		}
-	}
-}
-
-func swapEndianness(buf []byte) []byte {
-	for i := 0; i < len(buf)/2; i++ {
-		buf[i], buf[len(buf)-i-1] = buf[len(buf)-i-1], buf[i]
-	}
-	return buf
-}
-
-func TestBytesBigEquivalence(t *testing.T) {
-	f1 := func(in [32]byte, fe, fe1 Element) bool {
-		fe.SetBytes(in[:])
-
-		in[len(in)-1] &= (1 << 7) - 1 // mask the most significant bit
-		b := new(big.Int).SetBytes(swapEndianness(in[:]))
-		fe1.fromBig(b)
-
-		if fe != fe1 {
-			return false
-		}
-
-		buf := make([]byte, 32) // pad with zeroes
-		copy(buf, swapEndianness(fe1.toBig().Bytes()))
-
-		return bytes.Equal(fe.Bytes(), buf) && isInBounds(&fe) && isInBounds(&fe1)
-	}
-	if err := quick.Check(f1, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-// fromBig sets v = n, and returns v. The bit length of n must not exceed 256.
-func (v *Element) fromBig(n *big.Int) *Element {
-	if n.BitLen() > 32*8 {
-		panic("edwards25519: invalid field element input size")
-	}
-
-	buf := make([]byte, 0, 32)
-	for _, word := range n.Bits() {
-		for i := 0; i < bits.UintSize; i += 8 {
-			if len(buf) >= cap(buf) {
-				break
-			}
-			buf = append(buf, byte(word))
-			word >>= 8
-		}
-	}
-
-	return v.SetBytes(buf[:32])
-}
-
-func (v *Element) fromDecimal(s string) *Element {
-	n, ok := new(big.Int).SetString(s, 10)
-	if !ok {
-		panic("not a valid decimal: " + s)
-	}
-	return v.fromBig(n)
-}
-
-// toBig returns v as a big.Int.
-func (v *Element) toBig() *big.Int {
-	buf := v.Bytes()
-
-	words := make([]big.Word, 32*8/bits.UintSize)
-	for n := range words {
-		for i := 0; i < bits.UintSize; i += 8 {
-			if len(buf) == 0 {
-				break
-			}
-			words[n] |= big.Word(buf[0]) << big.Word(i)
-			buf = buf[1:]
-		}
-	}
-
-	return new(big.Int).SetBits(words)
-}
-
-func TestDecimalConstants(t *testing.T) {
-	sqrtM1String := "19681161376707505956807079304988542015446066515923890162744021073123829784752"
-	if exp := new(Element).fromDecimal(sqrtM1String); sqrtM1.Equal(exp) != 1 {
-		t.Errorf("sqrtM1 is %v, expected %v", sqrtM1, exp)
-	}
-	// d is in the parent package, and we don't want to expose d or fromDecimal.
-	// dString := "37095705934669439343138083508754565189542113879843219016388785533085940283555"
-	// if exp := new(Element).fromDecimal(dString); d.Equal(exp) != 1 {
-	// 	t.Errorf("d is %v, expected %v", d, exp)
-	// }
-}
-
-func TestSetBytesRoundTripEdgeCases(t *testing.T) {
-	// TODO: values close to 0, close to 2^255-19, between 2^255-19 and 2^255-1,
-	// and between 2^255 and 2^256-1. Test both the documented SetBytes
-	// behavior, and that Bytes reduces them.
-}
-
-// Tests self-consistency between Multiply and Square.
-func TestConsistency(t *testing.T) {
-	var x Element
-	var x2, x2sq Element
-
-	x = Element{1, 1, 1, 1, 1}
-	x2.Multiply(&x, &x)
-	x2sq.Square(&x)
-
-	if x2 != x2sq {
-		t.Fatalf("all ones failed\nmul: %x\nsqr: %x\n", x2, x2sq)
-	}
-
-	var bytes [32]byte
-
-	_, err := io.ReadFull(rand.Reader, bytes[:])
-	if err != nil {
-		t.Fatal(err)
-	}
-	x.SetBytes(bytes[:])
-
-	x2.Multiply(&x, &x)
-	x2sq.Square(&x)
-
-	if x2 != x2sq {
-		t.Fatalf("all ones failed\nmul: %x\nsqr: %x\n", x2, x2sq)
-	}
-}
-
-func TestEqual(t *testing.T) {
-	x := Element{1, 1, 1, 1, 1}
-	y := Element{5, 4, 3, 2, 1}
-
-	eq := x.Equal(&x)
-	if eq != 1 {
-		t.Errorf("wrong about equality")
-	}
-
-	eq = x.Equal(&y)
-	if eq != 0 {
-		t.Errorf("wrong about inequality")
-	}
-}
-
-func TestInvert(t *testing.T) {
-	x := Element{1, 1, 1, 1, 1}
-	one := Element{1, 0, 0, 0, 0}
-	var xinv, r Element
-
-	xinv.Invert(&x)
-	r.Multiply(&x, &xinv)
-	r.reduce()
-
-	if one != r {
-		t.Errorf("inversion identity failed, got: %x", r)
-	}
-
-	var bytes [32]byte
-
-	_, err := io.ReadFull(rand.Reader, bytes[:])
-	if err != nil {
-		t.Fatal(err)
-	}
-	x.SetBytes(bytes[:])
-
-	xinv.Invert(&x)
-	r.Multiply(&x, &xinv)
-	r.reduce()
-
-	if one != r {
-		t.Errorf("random inversion identity failed, got: %x for field element %x", r, x)
-	}
-
-	zero := Element{}
-	x.Set(&zero)
-	if xx := xinv.Invert(&x); xx != &xinv {
-		t.Errorf("inverting zero did not return the receiver")
-	} else if xinv.Equal(&zero) != 1 {
-		t.Errorf("inverting zero did not return zero")
-	}
-}
-
-func TestSelectSwap(t *testing.T) {
-	a := Element{358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676}
-	b := Element{84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972}
-
-	var c, d Element
-
-	c.Select(&a, &b, 1)
-	d.Select(&a, &b, 0)
-
-	if c.Equal(&a) != 1 || d.Equal(&b) != 1 {
-		t.Errorf("Select failed")
-	}
-
-	c.Swap(&d, 0)
-
-	if c.Equal(&a) != 1 || d.Equal(&b) != 1 {
-		t.Errorf("Swap failed")
-	}
-
-	c.Swap(&d, 1)
-
-	if c.Equal(&b) != 1 || d.Equal(&a) != 1 {
-		t.Errorf("Swap failed")
-	}
-}
-
-func TestMult32(t *testing.T) {
-	mult32EquivalentToMul := func(x Element, y uint32) bool {
-		t1 := new(Element)
-		for i := 0; i < 100; i++ {
-			t1.Mult32(&x, y)
-		}
-
-		ty := new(Element)
-		ty.l0 = uint64(y)
-
-		t2 := new(Element)
-		for i := 0; i < 100; i++ {
-			t2.Multiply(&x, ty)
-		}
-
-		return t1.Equal(t2) == 1 && isInBounds(t1) && isInBounds(t2)
-	}
-
-	if err := quick.Check(mult32EquivalentToMul, quickCheckConfig1024); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestSqrtRatio(t *testing.T) {
-	// From draft-irtf-cfrg-ristretto255-decaf448-00, Appendix A.4.
-	type test struct {
-		u, v      string
-		wasSquare int
-		r         string
-	}
-	var tests = []test{
-		// If u is 0, the function is defined to return (0, TRUE), even if v
-		// is zero. Note that where used in this package, the denominator v
-		// is never zero.
-		{
-			"0000000000000000000000000000000000000000000000000000000000000000",
-			"0000000000000000000000000000000000000000000000000000000000000000",
-			1, "0000000000000000000000000000000000000000000000000000000000000000",
-		},
-		// 0/1 == 0²
-		{
-			"0000000000000000000000000000000000000000000000000000000000000000",
-			"0100000000000000000000000000000000000000000000000000000000000000",
-			1, "0000000000000000000000000000000000000000000000000000000000000000",
-		},
-		// If u is non-zero and v is zero, defined to return (0, FALSE).
-		{
-			"0100000000000000000000000000000000000000000000000000000000000000",
-			"0000000000000000000000000000000000000000000000000000000000000000",
-			0, "0000000000000000000000000000000000000000000000000000000000000000",
-		},
-		// 2/1 is not square in this field.
-		{
-			"0200000000000000000000000000000000000000000000000000000000000000",
-			"0100000000000000000000000000000000000000000000000000000000000000",
-			0, "3c5ff1b5d8e4113b871bd052f9e7bcd0582804c266ffb2d4f4203eb07fdb7c54",
-		},
-		// 4/1 == 2²
-		{
-			"0400000000000000000000000000000000000000000000000000000000000000",
-			"0100000000000000000000000000000000000000000000000000000000000000",
-			1, "0200000000000000000000000000000000000000000000000000000000000000",
-		},
-		// 1/4 == (2⁻¹)² == (2^(p-2))² per Euler's theorem
-		{
-			"0100000000000000000000000000000000000000000000000000000000000000",
-			"0400000000000000000000000000000000000000000000000000000000000000",
-			1, "f6ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f",
-		},
-	}
-
-	for i, tt := range tests {
-		u := new(Element).SetBytes(decodeHex(tt.u))
-		v := new(Element).SetBytes(decodeHex(tt.v))
-		want := new(Element).SetBytes(decodeHex(tt.r))
-		got, wasSquare := new(Element).SqrtRatio(u, v)
-		if got.Equal(want) == 0 || wasSquare != tt.wasSquare {
-			t.Errorf("%d: got (%v, %v), want (%v, %v)", i, got, wasSquare, want, tt.wasSquare)
-		}
-	}
-}
-
-func TestCarryPropagate(t *testing.T) {
-	asmLikeGeneric := func(a [5]uint64) bool {
-		t1 := &Element{a[0], a[1], a[2], a[3], a[4]}
-		t2 := &Element{a[0], a[1], a[2], a[3], a[4]}
-
-		t1.carryPropagate()
-		t2.carryPropagateGeneric()
-
-		if *t1 != *t2 {
-			t.Logf("got: %#v,\nexpected: %#v", t1, t2)
-		}
-
-		return *t1 == *t2 && isInBounds(t2)
-	}
-
-	if err := quick.Check(asmLikeGeneric, quickCheckConfig1024); err != nil {
-		t.Error(err)
-	}
-
-	if !asmLikeGeneric([5]uint64{0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff}) {
-		t.Errorf("failed for {0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff}")
-	}
-}
-
-func TestFeSquare(t *testing.T) {
-	asmLikeGeneric := func(a Element) bool {
-		t1 := a
-		t2 := a
-
-		feSquareGeneric(&t1, &t1)
-		feSquare(&t2, &t2)
-
-		if t1 != t2 {
-			t.Logf("got: %#v,\nexpected: %#v", t1, t2)
-		}
-
-		return t1 == t2 && isInBounds(&t2)
-	}
-
-	if err := quick.Check(asmLikeGeneric, quickCheckConfig1024); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestFeMul(t *testing.T) {
-	asmLikeGeneric := func(a, b Element) bool {
-		a1 := a
-		a2 := a
-		b1 := b
-		b2 := b
-
-		feMulGeneric(&a1, &a1, &b1)
-		feMul(&a2, &a2, &b2)
-
-		if a1 != a2 || b1 != b2 {
-			t.Logf("got: %#v,\nexpected: %#v", a1, a2)
-			t.Logf("got: %#v,\nexpected: %#v", b1, b2)
-		}
-
-		return a1 == a2 && isInBounds(&a2) &&
-			b1 == b2 && isInBounds(&b2)
-	}
-
-	if err := quick.Check(asmLikeGeneric, quickCheckConfig1024); err != nil {
-		t.Error(err)
-	}
-}
-
-func decodeHex(s string) []byte {
-	b, err := hex.DecodeString(s)
-	if err != nil {
-		panic(err)
-	}
-	return b
-}

curve25519/internal/field/sync.checkpoint

@@ -1 +0,0 @@
-b0c49ae9f59d233526f8934262c5bbbe14d4358d

curve25519/internal/field/sync.sh

@@ -1,19 +0,0 @@
-#! /bin/bash
-set -euo pipefail
-
-cd "$(git rev-parse --show-toplevel)"
-
-STD_PATH=src/crypto/ed25519/internal/edwards25519/field
-LOCAL_PATH=curve25519/internal/field
-LAST_SYNC_REF=$(cat $LOCAL_PATH/sync.checkpoint)
-
-git fetch https://go.googlesource.com/go master
-
-if git diff --quiet $LAST_SYNC_REF:$STD_PATH FETCH_HEAD:$STD_PATH; then
-    echo "No changes."
-else
-    NEW_REF=$(git rev-parse FETCH_HEAD | tee $LOCAL_PATH/sync.checkpoint)
-    echo "Applying changes from $LAST_SYNC_REF to $NEW_REF..."
-    git diff $LAST_SYNC_REF:$STD_PATH FETCH_HEAD:$STD_PATH | \
-        git apply -3 --directory=$LOCAL_PATH
-fi

ed25519/ed25519.go

@@ -11,9 +11,7 @@
 // operations with the same key more efficient. This package refers to the RFC
 // 8032 private key as the “seed”.
 //
-// Beginning with Go 1.13, the functionality of this package was moved to the
-// standard library as crypto/ed25519. This package only acts as a compatibility
-// wrapper.
+// This package is a wrapper around the standard library crypto/ed25519 package.
 package ed25519
 
 import (

go.mod

@@ -1,11 +1,11 @@
 module golang.org/x/crypto
 
-go 1.18
+go 1.20
 
 require (
 	golang.org/x/net v0.21.0 // tagx:ignore
-	golang.org/x/sys v0.19.0
-	golang.org/x/term v0.19.0
+	golang.org/x/sys v0.25.0
+	golang.org/x/term v0.24.0
 )
 
-require golang.org/x/text v0.14.0 // indirect
+require golang.org/x/text v0.18.0 // indirect

go.sum

@@ -1,8 +1,8 @@
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
+golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=

hkdf/hkdf.go

@@ -8,7 +8,7 @@
 // HKDF is a cryptographic key derivation function (KDF) with the goal of
 // expanding limited input keying material into one or more cryptographically
 // strong secret keys.
-package hkdf // import "golang.org/x/crypto/hkdf"
+package hkdf
 
 import (
 	"crypto/hmac"

internal/poly1305/_asm/go.mod

@@ -0,0 +1,15 @@
+module internal/poly1305/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

internal/poly1305/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

internal/poly1305/_asm/sum_amd64_asm.go

@@ -0,0 +1,126 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/sha3"
+)
+
+//go:generate go run . -out ../sum_amd64.s -pkg poly1305
+
+func main() {
+	Package("golang.org/x/crypto/internal/poly1305")
+	ConstraintExpr("gc,!purego")
+	update()
+	Generate()
+}
+
+func update() {
+	Implement("update")
+
+	Load(Param("state"), RDI)
+	MOVQ(NewParamAddr("msg_base", 8), RSI)
+	MOVQ(NewParamAddr("msg_len", 16), R15)
+
+	MOVQ(Mem{Base: DI}.Offset(0), R8)   // h0
+	MOVQ(Mem{Base: DI}.Offset(8), R9)   // h1
+	MOVQ(Mem{Base: DI}.Offset(16), R10) // h2
+	MOVQ(Mem{Base: DI}.Offset(24), R11) // r0
+	MOVQ(Mem{Base: DI}.Offset(32), R12) // r1
+
+	CMPQ(R15, Imm(16))
+	JB(LabelRef("bytes_between_0_and_15"))
+
+	Label("loop")
+	POLY1305_ADD(RSI, R8, R9, R10)
+
+	Label("multiply")
+	POLY1305_MUL(R8, R9, R10, R11, R12, RBX, RCX, R13, R14)
+	SUBQ(Imm(16), R15)
+	CMPQ(R15, Imm(16))
+	JAE(LabelRef("loop"))
+
+	Label("bytes_between_0_and_15")
+	TESTQ(R15, R15)
+	JZ(LabelRef("done"))
+	MOVQ(U32(1), RBX)
+	XORQ(RCX, RCX)
+	XORQ(R13, R13)
+	ADDQ(R15, RSI)
+
+	Label("flush_buffer")
+	SHLQ(Imm(8), RBX, RCX)
+	SHLQ(Imm(8), RBX)
+	MOVB(Mem{Base: SI}.Offset(-1), R13B)
+	XORQ(R13, RBX)
+	DECQ(RSI)
+	DECQ(R15)
+	JNZ(LabelRef("flush_buffer"))
+
+	ADDQ(RBX, R8)
+	ADCQ(RCX, R9)
+	ADCQ(Imm(0), R10)
+	MOVQ(U32(16), R15)
+	JMP(LabelRef("multiply"))
+
+	Label("done")
+	MOVQ(R8, Mem{Base: DI}.Offset(0))
+	MOVQ(R9, Mem{Base: DI}.Offset(8))
+	MOVQ(R10, Mem{Base: DI}.Offset(16))
+	RET()
+}
+
+func POLY1305_ADD(msg, h0, h1, h2 GPPhysical) {
+	ADDQ(Mem{Base: msg}.Offset(0), h0)
+	ADCQ(Mem{Base: msg}.Offset(8), h1)
+	ADCQ(Imm(1), h2)
+	LEAQ(Mem{Base: msg}.Offset(16), msg)
+}
+
+func POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3 GPPhysical) {
+	MOVQ(r0, RAX)
+	MULQ(h0)
+	MOVQ(RAX, t0)
+	MOVQ(RDX, t1)
+	MOVQ(r0, RAX)
+	MULQ(h1)
+	ADDQ(RAX, t1)
+	ADCQ(Imm(0), RDX)
+	MOVQ(r0, t2)
+	IMULQ(h2, t2)
+	ADDQ(RDX, t2)
+
+	MOVQ(r1, RAX)
+	MULQ(h0)
+	ADDQ(RAX, t1)
+	ADCQ(Imm(0), RDX)
+	MOVQ(RDX, h0)
+	MOVQ(r1, t3)
+	IMULQ(h2, t3)
+	MOVQ(r1, RAX)
+	MULQ(h1)
+	ADDQ(RAX, t2)
+	ADCQ(RDX, t3)
+	ADDQ(h0, t2)
+	ADCQ(Imm(0), t3)
+
+	MOVQ(t0, h0)
+	MOVQ(t1, h1)
+	MOVQ(t2, h2)
+	ANDQ(Imm(3), h2)
+	MOVQ(t2, t0)
+	ANDQ(I32(-4), t0)
+	ADDQ(t0, h0)
+	ADCQ(t3, h1)
+	ADCQ(Imm(0), h2)
+	SHRQ(Imm(2), t3, t2)
+	SHRQ(Imm(2), t3)
+	ADDQ(t2, h0)
+	ADCQ(t3, h1)
+	ADCQ(Imm(0), h2)
+}

internal/poly1305/sum_amd64.s

@@ -1,108 +1,93 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run sum_amd64_asm.go -out ../sum_amd64.s -pkg poly1305. DO NOT EDIT.
 
 //go:build gc && !purego
 
-#include "textflag.h"
-
-#define POLY1305_ADD(msg, h0, h1, h2) \
-	ADDQ 0(msg), h0;  \
-	ADCQ 8(msg), h1;  \
-	ADCQ $1, h2;      \
-	LEAQ 16(msg), msg
-
-#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \
-	MOVQ  r0, AX;                  \
-	MULQ  h0;                      \
-	MOVQ  AX, t0;                  \
-	MOVQ  DX, t1;                  \
-	MOVQ  r0, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  r0, t2;                  \
-	IMULQ h2, t2;                  \
-	ADDQ  DX, t2;                  \
-	                               \
-	MOVQ  r1, AX;                  \
-	MULQ  h0;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  DX, h0;                  \
-	MOVQ  r1, t3;                  \
-	IMULQ h2, t3;                  \
-	MOVQ  r1, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t2;                  \
-	ADCQ  DX, t3;                  \
-	ADDQ  h0, t2;                  \
-	ADCQ  $0, t3;                  \
-	                               \
-	MOVQ  t0, h0;                  \
-	MOVQ  t1, h1;                  \
-	MOVQ  t2, h2;                  \
-	ANDQ  $3, h2;                  \
-	MOVQ  t2, t0;                  \
-	ANDQ  $0xFFFFFFFFFFFFFFFC, t0; \
-	ADDQ  t0, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2;                  \
-	SHRQ  $2, t3, t2;              \
-	SHRQ  $2, t3;                  \
-	ADDQ  t2, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2
-
-// func update(state *[7]uint64, msg []byte)
+// func update(state *macState, msg []byte)
 TEXT ·update(SB), $0-32
 	MOVQ state+0(FP), DI
 	MOVQ msg_base+8(FP), SI
 	MOVQ msg_len+16(FP), R15
-
-	MOVQ 0(DI), R8   // h0
-	MOVQ 8(DI), R9   // h1
-	MOVQ 16(DI), R10 // h2
-	MOVQ 24(DI), R11 // r0
-	MOVQ 32(DI), R12 // r1
-
-	CMPQ R15, $16
+	MOVQ (DI), R8
+	MOVQ 8(DI), R9
+	MOVQ 16(DI), R10
+	MOVQ 24(DI), R11
+	MOVQ 32(DI), R12
+	CMPQ R15, $0x10
 	JB   bytes_between_0_and_15
 
 loop:
-	POLY1305_ADD(SI, R8, R9, R10)
+	ADDQ (SI), R8
+	ADCQ 8(SI), R9
+	ADCQ $0x01, R10
+	LEAQ 16(SI), SI
 
 multiply:
-	POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14)
-	SUBQ $16, R15
-	CMPQ R15, $16
-	JAE  loop
+	MOVQ  R11, AX
+	MULQ  R8
+	MOVQ  AX, BX
+	MOVQ  DX, CX
+	MOVQ  R11, AX
+	MULQ  R9
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  R11, R13
+	IMULQ R10, R13
+	ADDQ  DX, R13
+	MOVQ  R12, AX
+	MULQ  R8
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  DX, R8
+	MOVQ  R12, R14
+	IMULQ R10, R14
+	MOVQ  R12, AX
+	MULQ  R9
+	ADDQ  AX, R13
+	ADCQ  DX, R14
+	ADDQ  R8, R13
+	ADCQ  $0x00, R14
+	MOVQ  BX, R8
+	MOVQ  CX, R9
+	MOVQ  R13, R10
+	ANDQ  $0x03, R10
+	MOVQ  R13, BX
+	ANDQ  $-4, BX
+	ADDQ  BX, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SHRQ  $0x02, R14, R13
+	SHRQ  $0x02, R14
+	ADDQ  R13, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SUBQ  $0x10, R15
+	CMPQ  R15, $0x10
+	JAE   loop
 
 bytes_between_0_and_15:
 	TESTQ R15, R15
 	JZ    done
-	MOVQ  $1, BX
+	MOVQ  $0x00000001, BX
 	XORQ  CX, CX
 	XORQ  R13, R13
 	ADDQ  R15, SI
 
 flush_buffer:
-	SHLQ $8, BX, CX
-	SHLQ $8, BX
+	SHLQ $0x08, BX, CX
+	SHLQ $0x08, BX
 	MOVB -1(SI), R13
 	XORQ R13, BX
 	DECQ SI
 	DECQ R15
 	JNZ  flush_buffer
-
 	ADDQ BX, R8
 	ADCQ CX, R9
-	ADCQ $0, R10
-	MOVQ $16, R15
+	ADCQ $0x00, R10
+	MOVQ $0x00000010, R15
 	JMP  multiply
 
 done:
-	MOVQ R8, 0(DI)
+	MOVQ R8, (DI)
 	MOVQ R9, 8(DI)
 	MOVQ R10, 16(DI)
 	RET

internal/wycheproof/ecdh_stdlib_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build go1.20
-
 package wycheproof
 
 import (

internal/wycheproof/eddsa_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build go1.13
-
 package wycheproof
 
 import (

md4/md4.go

@@ -7,7 +7,7 @@
 // Deprecated: MD4 is cryptographically broken and should only be used
 // where compatibility with legacy systems, not security, is the goal. Instead,
 // use a secure hash like SHA-256 (from crypto/sha256).
-package md4 // import "golang.org/x/crypto/md4"
+package md4
 
 import (
 	"crypto"

nacl/box/box.go

@@ -35,7 +35,7 @@ Anonymous sealing/opening is an extension of NaCl defined by and interoperable
 with libsodium:
 https://libsodium.gitbook.io/doc/public-key_cryptography/sealed_boxes.
 */
-package box // import "golang.org/x/crypto/nacl/box"
+package box
 
 import (
 	cryptorand "crypto/rand"

nacl/secretbox/secretbox.go

@@ -32,7 +32,7 @@ chunk size.
 
 This package is interoperable with NaCl: https://nacl.cr.yp.to/secretbox.html.
 */
-package secretbox // import "golang.org/x/crypto/nacl/secretbox"
+package secretbox
 
 import (
 	"golang.org/x/crypto/internal/alias"

ocsp/ocsp.go

@@ -5,7 +5,7 @@
 // Package ocsp parses OCSP responses as specified in RFC 2560. OCSP responses
 // are signed messages attesting to the validity of a certificate for a small
 // period of time. This is used to manage revocation for X.509 certificates.
-package ocsp // import "golang.org/x/crypto/ocsp"
+package ocsp
 
 import (
 	"crypto"

ocsp/ocsp_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build go1.7
-
 package ocsp
 
 import (

openpgp/armor/armor.go

@@ -10,14 +10,15 @@
 // for their specific task. If you are required to interoperate with OpenPGP
 // systems and need a maintained package, consider a community fork.
 // See https://golang.org/issue/44226.
-package armor // import "golang.org/x/crypto/openpgp/armor"
+package armor
 
 import (
 	"bufio"
 	"bytes"
 	"encoding/base64"
-	"golang.org/x/crypto/openpgp/errors"
 	"io"
+
+	"golang.org/x/crypto/openpgp/errors"
 )
 
 // A Block represents an OpenPGP armored structure.

openpgp/clearsign/clearsign.go

@@ -13,7 +13,7 @@
 // for their specific task. If you are required to interoperate with OpenPGP
 // systems and need a maintained package, consider a community fork.
 // See https://golang.org/issue/44226.
-package clearsign // import "golang.org/x/crypto/openpgp/clearsign"
+package clearsign
 
 import (
 	"bufio"

openpgp/elgamal/elgamal.go

@@ -16,7 +16,7 @@
 // https://golang.org/issue/44226), and ElGamal in the OpenPGP ecosystem has
 // compatibility and security issues (see https://eprint.iacr.org/2021/923).
 // Moreover, this package doesn't protect against side-channel attacks.
-package elgamal // import "golang.org/x/crypto/openpgp/elgamal"
+package elgamal
 
 import (
 	"crypto/rand"

openpgp/errors/errors.go

@@ -9,7 +9,7 @@
 // for their specific task. If you are required to interoperate with OpenPGP
 // systems and need a maintained package, consider a community fork.
 // See https://golang.org/issue/44226.
-package errors // import "golang.org/x/crypto/openpgp/errors"
+package errors
 
 import (
 	"strconv"

openpgp/keys_test.go

@@ -132,7 +132,7 @@ func TestRevokedUserID(t *testing.T) {
 	}
 }
 
-// TestExternallyRevokableKey attempts to load and parse a key with a third party revocation permission.
+// TestExternallyRevocableKey attempts to load and parse a key with a third party revocation permission.
 func TestExternallyRevocableKey(t *testing.T) {
 	kring, err := ReadKeyRing(readerFromHex(subkeyUsageHex))
 	if err != nil {

openpgp/packet/packet.go

@@ -10,7 +10,7 @@
 // for their specific task. If you are required to interoperate with OpenPGP
 // systems and need a maintained package, consider a community fork.
 // See https://golang.org/issue/44226.
-package packet // import "golang.org/x/crypto/openpgp/packet"
+package packet
 
 import (
 	"bufio"

openpgp/read.go

@@ -9,7 +9,7 @@
 // for their specific task. If you are required to interoperate with OpenPGP
 // systems and need a maintained package, consider a community fork.
 // See https://golang.org/issue/44226.
-package openpgp // import "golang.org/x/crypto/openpgp"
+package openpgp
 
 import (
 	"crypto"

openpgp/s2k/s2k.go

@@ -10,7 +10,7 @@
 // for their specific task. If you are required to interoperate with OpenPGP
 // systems and need a maintained package, consider a community fork.
 // See https://golang.org/issue/44226.
-package s2k // import "golang.org/x/crypto/openpgp/s2k"
+package s2k
 
 import (
 	"crypto"

otr/otr.go

@@ -8,7 +8,7 @@
 // The version of OTR implemented by this package has been deprecated
 // (https://bugs.otr.im/lib/libotr/issues/140). An implementation of OTRv3 is
 // available at https://github.com/coyim/otr3.
-package otr // import "golang.org/x/crypto/otr"
+package otr
 
 import (
 	"bytes"

pbkdf2/pbkdf2.go

@@ -16,7 +16,7 @@ Hash Functions SHA-1, SHA-224, SHA-256, SHA-384 and SHA-512 for HMAC. To
 choose, you can pass the `New` functions from the different SHA packages to
 pbkdf2.Key.
 */
-package pbkdf2 // import "golang.org/x/crypto/pbkdf2"
+package pbkdf2
 
 import (
 	"crypto/hmac"

poly1305/poly1305_compat.go

@@ -21,7 +21,7 @@
 // For encryption, use the full ChaCha20-Poly1305 construction implemented by
 // golang.org/x/crypto/chacha20poly1305. For authentication, use a general
 // purpose MAC such as HMAC implemented by crypto/hmac.
-package poly1305 // import "golang.org/x/crypto/poly1305"
+package poly1305
 
 import "golang.org/x/crypto/internal/poly1305"
 

ripemd160/ripemd160.go

@@ -7,7 +7,7 @@
 // Deprecated: RIPEMD-160 is a legacy hash and should not be used for new
 // applications. Also, this package does not and will not provide an optimized
 // implementation. Instead, use a modern hash like SHA-256 (from crypto/sha256).
-package ripemd160 // import "golang.org/x/crypto/ripemd160"
+package ripemd160
 
 // RIPEMD-160 is designed by Hans Dobbertin, Antoon Bosselaers, and Bart
 // Preneel with specifications available at:

salsa20/salsa/_asm/go.mod

@@ -0,0 +1,14 @@
+module salsa20/salsa/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)

salsa20/salsa/_asm/go.sum

@@ -0,0 +1,10 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=

salsa20/salsa/_asm/salsa20_amd64_asm.go

@@ -0,0 +1,932 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	"github.com/mmcloughlin/avo/ir"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/salsa20/salsa"
+)
+
+//go:generate go run . -out ../salsa20_amd64.s -pkg salsa
+
+func main() {
+	Package("golang.org/x/crypto/salsa20/salsa")
+	ConstraintExpr("amd64,!purego,gc")
+	salsa2020XORKeyStream()
+	Generate()
+}
+
+func salsa2020XORKeyStream() {
+	Implement("salsa2020XORKeyStream")
+	Attributes(0)
+	AllocLocal(456) // frame = 424 + 32 byte alignment
+	Comment("This needs up to 64 bytes at 360(R12); hence the non-obvious frame size.")
+
+	Load(Param("out"), RDI)
+	Load(Param("in"), RSI)
+	Load(Param("n"), RDX)
+	Load(Param("nonce"), RCX)
+	Load(Param("key"), R8)
+
+	MOVQ(RSP, R12)
+	ADDQ(Imm(31), R12)
+	ANDQ(I32(^31), R12)
+
+	MOVQ(RDX, R9)
+	MOVQ(RCX, RDX)
+	MOVQ(R8, R10)
+	CMPQ(R9, Imm(0))
+	JBE(LabelRef("DONE"))
+
+	START()
+	BYTESATLEAST256()
+	MAINLOOP1()
+	BYTESBETWEEN1AND255()
+	NOCOPY()
+	MAINLOOP2()
+
+	Label("BYTESATLEAST64")
+	Label("DONE")
+	RET()
+	Label("BYTESATLEAST65")
+	SUBQ(Imm(64), R9)
+	ADDQ(Imm(64), RDI)
+	ADDQ(Imm(64), RSI)
+	JMP(LabelRef("BYTESBETWEEN1AND255"))
+}
+
+func START() {
+	Label("START")
+	MOVL(Mem{Base: R10}.Offset(20), ECX)
+	MOVL(Mem{Base: R10}.Offset(0), R8L)
+	MOVL(Mem{Base: EDX}.Offset(0), EAX)
+	MOVL(Mem{Base: R10}.Offset(16), R11L)
+	MOVL(ECX, Mem{Base: R12}.Offset(0))
+	MOVL(R8L, Mem{Base: R12}.Offset(4))
+	MOVL(EAX, Mem{Base: R12}.Offset(8))
+	MOVL(R11L, Mem{Base: R12}.Offset(12))
+	MOVL(Mem{Base: EDX}.Offset(8), ECX)
+	MOVL(Mem{Base: R10}.Offset(24), R8L)
+	MOVL(Mem{Base: R10}.Offset(4), EAX)
+	MOVL(Mem{Base: EDX}.Offset(4), R11L)
+	MOVL(ECX, Mem{Base: R12}.Offset(16))
+	MOVL(R8L, Mem{Base: R12}.Offset(20))
+	MOVL(EAX, Mem{Base: R12}.Offset(24))
+	MOVL(R11L, Mem{Base: R12}.Offset(28))
+	MOVL(Mem{Base: EDX}.Offset(12), ECX)
+	MOVL(Mem{Base: R10}.Offset(12), EDX)
+	MOVL(Mem{Base: R10}.Offset(28), R8L)
+	MOVL(Mem{Base: R10}.Offset(8), EAX)
+	MOVL(EDX, Mem{Base: R12}.Offset(32))
+	MOVL(ECX, Mem{Base: R12}.Offset(36))
+	MOVL(R8L, Mem{Base: R12}.Offset(40))
+	MOVL(EAX, Mem{Base: R12}.Offset(44))
+	MOVQ(Imm(1634760805), RDX)
+	MOVQ(Imm(857760878), RCX)
+	MOVQ(Imm(2036477234), R8)
+	MOVQ(Imm(1797285236), RAX)
+	MOVL(EDX, Mem{Base: R12}.Offset(48))
+	MOVL(ECX, Mem{Base: R12}.Offset(52))
+	MOVL(R8L, Mem{Base: R12}.Offset(56))
+	MOVL(EAX, Mem{Base: R12}.Offset(60))
+	CMPQ(R9, U32(256))
+	JB(LabelRef("BYTESBETWEEN1AND255"))
+	MOVOA(Mem{Base: R12}.Offset(48), X0)
+	PSHUFL(Imm(0x55), X0, X1)
+	PSHUFL(Imm(0xAA), X0, X2)
+	PSHUFL(Imm(0xFF), X0, X3)
+	PSHUFL(Imm(0x00), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(64))
+	MOVOA(X2, Mem{Base: R12}.Offset(80))
+	MOVOA(X3, Mem{Base: R12}.Offset(96))
+	MOVOA(X0, Mem{Base: R12}.Offset(112))
+	MOVOA(Mem{Base: R12}.Offset(0), X0)
+	PSHUFL(Imm(0xAA), X0, X1)
+	PSHUFL(Imm(0xFF), X0, X2)
+	PSHUFL(Imm(0x00), X0, X3)
+	PSHUFL(Imm(0x55), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(128))
+	MOVOA(X2, Mem{Base: R12}.Offset(144))
+	MOVOA(X3, Mem{Base: R12}.Offset(160))
+	MOVOA(X0, Mem{Base: R12}.Offset(176))
+	MOVOA(Mem{Base: R12}.Offset(16), X0)
+	PSHUFL(Imm(0xFF), X0, X1)
+	PSHUFL(Imm(0x55), X0, X2)
+	PSHUFL(Imm(0xAA), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(192))
+	MOVOA(X2, Mem{Base: R12}.Offset(208))
+	MOVOA(X0, Mem{Base: R12}.Offset(224))
+	MOVOA(Mem{Base: R12}.Offset(32), X0)
+	PSHUFL(Imm(0x00), X0, X1)
+	PSHUFL(Imm(0xAA), X0, X2)
+	PSHUFL(Imm(0xFF), X0, X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(240))
+	MOVOA(X2, Mem{Base: R12}.Offset(256))
+	MOVOA(X0, Mem{Base: R12}.Offset(272))
+
+}
+
+func BYTESATLEAST256() {
+	Label("BYTESATLEAST256")
+	MOVL(Mem{Base: R12}.Offset(16), EDX)
+	MOVL(Mem{Base: R12}.Offset(36), ECX)
+	MOVL(EDX, Mem{Base: R12}.Offset(288))
+	MOVL(ECX, Mem{Base: R12}.Offset(304))
+	SHLQ(Imm(32), RCX)
+	ADDQ(RCX, RDX)
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(292))
+	MOVL(ECX, Mem{Base: R12}.Offset(308))
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(296))
+	MOVL(ECX, Mem{Base: R12}.Offset(312))
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(300))
+	MOVL(ECX, Mem{Base: R12}.Offset(316))
+	ADDQ(Imm(1), RDX)
+	MOVQ(RDX, RCX)
+	SHRQ(Imm(32), RCX)
+	MOVL(EDX, Mem{Base: R12}.Offset(16))
+	MOVL(ECX, Mem{Base: R12}.Offset(36))
+	MOVQ(R9, Mem{Base: R12}.Offset(352))
+	MOVQ(U32(20), RDX)
+	MOVOA(Mem{Base: R12}.Offset(64), X0)
+	MOVOA(Mem{Base: R12}.Offset(80), X1)
+	MOVOA(Mem{Base: R12}.Offset(96), X2)
+	MOVOA(Mem{Base: R12}.Offset(256), X3)
+	MOVOA(Mem{Base: R12}.Offset(272), X4)
+	MOVOA(Mem{Base: R12}.Offset(128), X5)
+	MOVOA(Mem{Base: R12}.Offset(144), X6)
+	MOVOA(Mem{Base: R12}.Offset(176), X7)
+	MOVOA(Mem{Base: R12}.Offset(192), X8)
+	MOVOA(Mem{Base: R12}.Offset(208), X9)
+	MOVOA(Mem{Base: R12}.Offset(224), X10)
+	MOVOA(Mem{Base: R12}.Offset(304), X11)
+	MOVOA(Mem{Base: R12}.Offset(112), X12)
+	MOVOA(Mem{Base: R12}.Offset(160), X13)
+	MOVOA(Mem{Base: R12}.Offset(240), X14)
+	MOVOA(Mem{Base: R12}.Offset(288), X15)
+}
+
+func MAINLOOP1() {
+	Label("MAINLOOP1")
+	MOVOA(X1, Mem{Base: R12}.Offset(320))
+	MOVOA(X2, Mem{Base: R12}.Offset(336))
+	MOVOA(X13, X1)
+	PADDL(X12, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X14)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X14)
+	MOVOA(X7, X1)
+	PADDL(X0, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X11)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X11)
+	MOVOA(X12, X1)
+	PADDL(X14, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X15)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X15)
+	MOVOA(X0, X1)
+	PADDL(X11, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X9)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X9)
+	MOVOA(X14, X1)
+	PADDL(X15, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X13)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X13)
+	MOVOA(X11, X1)
+	PADDL(X9, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X7)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X7)
+	MOVOA(X15, X1)
+	PADDL(X13, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(18), X1)
+	PXOR(X1, X12)
+	PSRLL(Imm(14), X2)
+	PXOR(X2, X12)
+	MOVOA(Mem{Base: R12}.Offset(320), X1)
+	MOVOA(X12, Mem{Base: R12}.Offset(320))
+	MOVOA(X9, X2)
+	PADDL(X7, X2)
+	MOVOA(X2, X12)
+	PSLLL(Imm(18), X2)
+	PXOR(X2, X0)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X0)
+	MOVOA(X5, X2)
+	PADDL(X1, X2)
+	MOVOA(X2, X12)
+	PSLLL(Imm(7), X2)
+	PXOR(X2, X3)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X3)
+	MOVOA(Mem{Base: R12}.Offset(336), X2)
+	MOVOA(X0, Mem{Base: R12}.Offset(336))
+	MOVOA(X6, X0)
+	PADDL(X2, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(7), X0)
+	PXOR(X0, X4)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X4)
+	MOVOA(X1, X0)
+	PADDL(X3, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X10)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X10)
+	MOVOA(X2, X0)
+	PADDL(X4, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X8)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X8)
+	MOVOA(X3, X0)
+	PADDL(X10, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X5)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X5)
+	MOVOA(X4, X0)
+	PADDL(X8, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X6)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X6)
+	MOVOA(X10, X0)
+	PADDL(X5, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X1)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X1)
+	MOVOA(Mem{Base: R12}.Offset(320), X0)
+	MOVOA(X1, Mem{Base: R12}.Offset(320))
+	MOVOA(X4, X1)
+	PADDL(X0, X1)
+	MOVOA(X1, X12)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X7)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X7)
+	MOVOA(X8, X1)
+	PADDL(X6, X1)
+	MOVOA(X1, X12)
+	PSLLL(Imm(18), X1)
+	PXOR(X1, X2)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X2)
+	MOVOA(Mem{Base: R12}.Offset(336), X12)
+	MOVOA(X2, Mem{Base: R12}.Offset(336))
+	MOVOA(X14, X1)
+	PADDL(X12, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(7), X1)
+	PXOR(X1, X5)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X5)
+	MOVOA(X0, X1)
+	PADDL(X7, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X10)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X10)
+	MOVOA(X12, X1)
+	PADDL(X5, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(9), X1)
+	PXOR(X1, X8)
+	PSRLL(Imm(23), X2)
+	PXOR(X2, X8)
+	MOVOA(X7, X1)
+	PADDL(X10, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X4)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X4)
+	MOVOA(X5, X1)
+	PADDL(X8, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(13), X1)
+	PXOR(X1, X14)
+	PSRLL(Imm(19), X2)
+	PXOR(X2, X14)
+	MOVOA(X10, X1)
+	PADDL(X4, X1)
+	MOVOA(X1, X2)
+	PSLLL(Imm(18), X1)
+	PXOR(X1, X0)
+	PSRLL(Imm(14), X2)
+	PXOR(X2, X0)
+	MOVOA(Mem{Base: R12}.Offset(320), X1)
+	MOVOA(X0, Mem{Base: R12}.Offset(320))
+	MOVOA(X8, X0)
+	PADDL(X14, X0)
+	MOVOA(X0, X2)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X12)
+	PSRLL(Imm(14), X2)
+	PXOR(X2, X12)
+	MOVOA(X11, X0)
+	PADDL(X1, X0)
+	MOVOA(X0, X2)
+	PSLLL(Imm(7), X0)
+	PXOR(X0, X6)
+	PSRLL(Imm(25), X2)
+	PXOR(X2, X6)
+	MOVOA(Mem{Base: R12}.Offset(336), X2)
+	MOVOA(X12, Mem{Base: R12}.Offset(336))
+	MOVOA(X3, X0)
+	PADDL(X2, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(7), X0)
+	PXOR(X0, X13)
+	PSRLL(Imm(25), X12)
+	PXOR(X12, X13)
+	MOVOA(X1, X0)
+	PADDL(X6, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X15)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X15)
+	MOVOA(X2, X0)
+	PADDL(X13, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(9), X0)
+	PXOR(X0, X9)
+	PSRLL(Imm(23), X12)
+	PXOR(X12, X9)
+	MOVOA(X6, X0)
+	PADDL(X15, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X11)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X11)
+	MOVOA(X13, X0)
+	PADDL(X9, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(13), X0)
+	PXOR(X0, X3)
+	PSRLL(Imm(19), X12)
+	PXOR(X12, X3)
+	MOVOA(X15, X0)
+	PADDL(X11, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X1)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X1)
+	MOVOA(X9, X0)
+	PADDL(X3, X0)
+	MOVOA(X0, X12)
+	PSLLL(Imm(18), X0)
+	PXOR(X0, X2)
+	PSRLL(Imm(14), X12)
+	PXOR(X12, X2)
+	MOVOA(Mem{Base: R12}.Offset(320), X12)
+	MOVOA(Mem{Base: R12}.Offset(336), X0)
+	SUBQ(Imm(2), RDX)
+	JA(LabelRef("MAINLOOP1"))
+	PADDL(Mem{Base: R12}.Offset(112), X12)
+	PADDL(Mem{Base: R12}.Offset(176), X7)
+	PADDL(Mem{Base: R12}.Offset(224), X10)
+	PADDL(Mem{Base: R12}.Offset(272), X4)
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	PSHUFL(Imm(0x39), X12, X12)
+	PSHUFL(Imm(0x39), X7, X7)
+	PSHUFL(Imm(0x39), X10, X10)
+	PSHUFL(Imm(0x39), X4, X4)
+	XORL(Mem{Base: SI}.Offset(0), EDX)
+	XORL(Mem{Base: SI}.Offset(4), ECX)
+	XORL(Mem{Base: SI}.Offset(8), R8L)
+	XORL(Mem{Base: SI}.Offset(12), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(0))
+	MOVL(ECX, Mem{Base: DI}.Offset(4))
+	MOVL(R8L, Mem{Base: DI}.Offset(8))
+	MOVL(R9L, Mem{Base: DI}.Offset(12))
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	PSHUFL(Imm(0x39), X12, X12)
+	PSHUFL(Imm(0x39), X7, X7)
+	PSHUFL(Imm(0x39), X10, X10)
+	PSHUFL(Imm(0x39), X4, X4)
+	XORL(Mem{Base: SI}.Offset(64), EDX)
+	XORL(Mem{Base: SI}.Offset(68), ECX)
+	XORL(Mem{Base: SI}.Offset(72), R8L)
+	XORL(Mem{Base: SI}.Offset(76), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(64))
+	MOVL(ECX, Mem{Base: DI}.Offset(68))
+	MOVL(R8L, Mem{Base: DI}.Offset(72))
+	MOVL(R9L, Mem{Base: DI}.Offset(76))
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	PSHUFL(Imm(0x39), X12, X12)
+	PSHUFL(Imm(0x39), X7, X7)
+	PSHUFL(Imm(0x39), X10, X10)
+	PSHUFL(Imm(0x39), X4, X4)
+	XORL(Mem{Base: SI}.Offset(128), EDX)
+	XORL(Mem{Base: SI}.Offset(132), ECX)
+	XORL(Mem{Base: SI}.Offset(136), R8L)
+	XORL(Mem{Base: SI}.Offset(140), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(128))
+	MOVL(ECX, Mem{Base: DI}.Offset(132))
+	MOVL(R8L, Mem{Base: DI}.Offset(136))
+	MOVL(R9L, Mem{Base: DI}.Offset(140))
+	MOVD(X12, EDX)
+	MOVD(X7, ECX)
+	MOVD(X10, R8)
+	MOVD(X4, R9)
+	XORL(Mem{Base: SI}.Offset(192), EDX)
+	XORL(Mem{Base: SI}.Offset(196), ECX)
+	XORL(Mem{Base: SI}.Offset(200), R8L)
+	XORL(Mem{Base: SI}.Offset(204), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(192))
+	MOVL(ECX, Mem{Base: DI}.Offset(196))
+	MOVL(R8L, Mem{Base: DI}.Offset(200))
+	MOVL(R9L, Mem{Base: DI}.Offset(204))
+	PADDL(Mem{Base: R12}.Offset(240), X14)
+	PADDL(Mem{Base: R12}.Offset(64), X0)
+	PADDL(Mem{Base: R12}.Offset(128), X5)
+	PADDL(Mem{Base: R12}.Offset(192), X8)
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	PSHUFL(Imm(0x39), X14, X14)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X5, X5)
+	PSHUFL(Imm(0x39), X8, X8)
+	XORL(Mem{Base: SI}.Offset(16), EDX)
+	XORL(Mem{Base: SI}.Offset(20), ECX)
+	XORL(Mem{Base: SI}.Offset(24), R8L)
+	XORL(Mem{Base: SI}.Offset(28), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(16))
+	MOVL(ECX, Mem{Base: DI}.Offset(20))
+	MOVL(R8L, Mem{Base: DI}.Offset(24))
+	MOVL(R9L, Mem{Base: DI}.Offset(28))
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	PSHUFL(Imm(0x39), X14, X14)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X5, X5)
+	PSHUFL(Imm(0x39), X8, X8)
+	XORL(Mem{Base: SI}.Offset(80), EDX)
+	XORL(Mem{Base: SI}.Offset(84), ECX)
+	XORL(Mem{Base: SI}.Offset(88), R8L)
+	XORL(Mem{Base: SI}.Offset(92), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(80))
+	MOVL(ECX, Mem{Base: DI}.Offset(84))
+	MOVL(R8L, Mem{Base: DI}.Offset(88))
+	MOVL(R9L, Mem{Base: DI}.Offset(92))
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	PSHUFL(Imm(0x39), X14, X14)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X5, X5)
+	PSHUFL(Imm(0x39), X8, X8)
+	XORL(Mem{Base: SI}.Offset(144), EDX)
+	XORL(Mem{Base: SI}.Offset(148), ECX)
+	XORL(Mem{Base: SI}.Offset(152), R8L)
+	XORL(Mem{Base: SI}.Offset(156), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(144))
+	MOVL(ECX, Mem{Base: DI}.Offset(148))
+	MOVL(R8L, Mem{Base: DI}.Offset(152))
+	MOVL(R9L, Mem{Base: DI}.Offset(156))
+	MOVD(X14, EDX)
+	MOVD(X0, ECX)
+	MOVD(X5, R8)
+	MOVD(X8, R9)
+	XORL(Mem{Base: SI}.Offset(208), EDX)
+	XORL(Mem{Base: SI}.Offset(212), ECX)
+	XORL(Mem{Base: SI}.Offset(216), R8L)
+	XORL(Mem{Base: SI}.Offset(220), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(208))
+	MOVL(ECX, Mem{Base: DI}.Offset(212))
+	MOVL(R8L, Mem{Base: DI}.Offset(216))
+	MOVL(R9L, Mem{Base: DI}.Offset(220))
+	PADDL(Mem{Base: R12}.Offset(288), X15)
+	PADDL(Mem{Base: R12}.Offset(304), X11)
+	PADDL(Mem{Base: R12}.Offset(80), X1)
+	PADDL(Mem{Base: R12}.Offset(144), X6)
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	PSHUFL(Imm(0x39), X15, X15)
+	PSHUFL(Imm(0x39), X11, X11)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X6, X6)
+	XORL(Mem{Base: SI}.Offset(32), EDX)
+	XORL(Mem{Base: SI}.Offset(36), ECX)
+	XORL(Mem{Base: SI}.Offset(40), R8L)
+	XORL(Mem{Base: SI}.Offset(44), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(32))
+	MOVL(ECX, Mem{Base: DI}.Offset(36))
+	MOVL(R8L, Mem{Base: DI}.Offset(40))
+	MOVL(R9L, Mem{Base: DI}.Offset(44))
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	PSHUFL(Imm(0x39), X15, X15)
+	PSHUFL(Imm(0x39), X11, X11)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X6, X6)
+	XORL(Mem{Base: SI}.Offset(96), EDX)
+	XORL(Mem{Base: SI}.Offset(100), ECX)
+	XORL(Mem{Base: SI}.Offset(104), R8L)
+	XORL(Mem{Base: SI}.Offset(108), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(96))
+	MOVL(ECX, Mem{Base: DI}.Offset(100))
+	MOVL(R8L, Mem{Base: DI}.Offset(104))
+	MOVL(R9L, Mem{Base: DI}.Offset(108))
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	PSHUFL(Imm(0x39), X15, X15)
+	PSHUFL(Imm(0x39), X11, X11)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X6, X6)
+	XORL(Mem{Base: SI}.Offset(160), EDX)
+	XORL(Mem{Base: SI}.Offset(164), ECX)
+	XORL(Mem{Base: SI}.Offset(168), R8L)
+	XORL(Mem{Base: SI}.Offset(172), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(160))
+	MOVL(ECX, Mem{Base: DI}.Offset(164))
+	MOVL(R8L, Mem{Base: DI}.Offset(168))
+	MOVL(R9L, Mem{Base: DI}.Offset(172))
+	MOVD(X15, EDX)
+	MOVD(X11, ECX)
+	MOVD(X1, R8)
+	MOVD(X6, R9)
+	XORL(Mem{Base: SI}.Offset(224), EDX)
+	XORL(Mem{Base: SI}.Offset(228), ECX)
+	XORL(Mem{Base: SI}.Offset(232), R8L)
+	XORL(Mem{Base: SI}.Offset(236), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(224))
+	MOVL(ECX, Mem{Base: DI}.Offset(228))
+	MOVL(R8L, Mem{Base: DI}.Offset(232))
+	MOVL(R9L, Mem{Base: DI}.Offset(236))
+	PADDL(Mem{Base: R12}.Offset(160), X13)
+	PADDL(Mem{Base: R12}.Offset(208), X9)
+	PADDL(Mem{Base: R12}.Offset(256), X3)
+	PADDL(Mem{Base: R12}.Offset(96), X2)
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	PSHUFL(Imm(0x39), X13, X13)
+	PSHUFL(Imm(0x39), X9, X9)
+	PSHUFL(Imm(0x39), X3, X3)
+	PSHUFL(Imm(0x39), X2, X2)
+	XORL(Mem{Base: SI}.Offset(48), EDX)
+	XORL(Mem{Base: SI}.Offset(52), ECX)
+	XORL(Mem{Base: SI}.Offset(56), R8L)
+	XORL(Mem{Base: SI}.Offset(60), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(48))
+	MOVL(ECX, Mem{Base: DI}.Offset(52))
+	MOVL(R8L, Mem{Base: DI}.Offset(56))
+	MOVL(R9L, Mem{Base: DI}.Offset(60))
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	PSHUFL(Imm(0x39), X13, X13)
+	PSHUFL(Imm(0x39), X9, X9)
+	PSHUFL(Imm(0x39), X3, X3)
+	PSHUFL(Imm(0x39), X2, X2)
+	XORL(Mem{Base: SI}.Offset(112), EDX)
+	XORL(Mem{Base: SI}.Offset(116), ECX)
+	XORL(Mem{Base: SI}.Offset(120), R8L)
+	XORL(Mem{Base: SI}.Offset(124), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(112))
+	MOVL(ECX, Mem{Base: DI}.Offset(116))
+	MOVL(R8L, Mem{Base: DI}.Offset(120))
+	MOVL(R9L, Mem{Base: DI}.Offset(124))
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	PSHUFL(Imm(0x39), X13, X13)
+	PSHUFL(Imm(0x39), X9, X9)
+	PSHUFL(Imm(0x39), X3, X3)
+	PSHUFL(Imm(0x39), X2, X2)
+	XORL(Mem{Base: SI}.Offset(176), EDX)
+	XORL(Mem{Base: SI}.Offset(180), ECX)
+	XORL(Mem{Base: SI}.Offset(184), R8L)
+	XORL(Mem{Base: SI}.Offset(188), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(176))
+	MOVL(ECX, Mem{Base: DI}.Offset(180))
+	MOVL(R8L, Mem{Base: DI}.Offset(184))
+	MOVL(R9L, Mem{Base: DI}.Offset(188))
+	MOVD(X13, EDX)
+	MOVD(X9, ECX)
+	MOVD(X3, R8)
+	MOVD(X2, R9)
+	XORL(Mem{Base: SI}.Offset(240), EDX)
+	XORL(Mem{Base: SI}.Offset(244), ECX)
+	XORL(Mem{Base: SI}.Offset(248), R8L)
+	XORL(Mem{Base: SI}.Offset(252), R9L)
+	MOVL(EDX, Mem{Base: DI}.Offset(240))
+	MOVL(ECX, Mem{Base: DI}.Offset(244))
+	MOVL(R8L, Mem{Base: DI}.Offset(248))
+	MOVL(R9L, Mem{Base: DI}.Offset(252))
+	MOVQ(Mem{Base: R12}.Offset(352), R9)
+	SUBQ(U32(256), R9)
+	ADDQ(U32(256), RSI)
+	ADDQ(U32(256), RDI)
+	CMPQ(R9, U32(256))
+	JAE(LabelRef("BYTESATLEAST256"))
+	CMPQ(R9, Imm(0))
+	JBE(LabelRef("DONE"))
+}
+
+func BYTESBETWEEN1AND255() {
+	Label("BYTESBETWEEN1AND255")
+	CMPQ(R9, Imm(64))
+	JAE(LabelRef("NOCOPY"))
+	MOVQ(RDI, RDX)
+	LEAQ(Mem{Base: R12}.Offset(360), RDI)
+	MOVQ(R9, RCX)
+	// Hack to get Avo to emit:
+	// 	REP; MOVSB
+	Instruction(&ir.Instruction{Opcode: "REP; MOVSB"})
+	LEAQ(Mem{Base: R12}.Offset(360), RDI)
+	LEAQ(Mem{Base: R12}.Offset(360), RSI)
+}
+
+func NOCOPY() {
+	Label("NOCOPY")
+	MOVQ(R9, Mem{Base: R12}.Offset(352))
+	MOVOA(Mem{Base: R12}.Offset(48), X0)
+	MOVOA(Mem{Base: R12}.Offset(0), X1)
+	MOVOA(Mem{Base: R12}.Offset(16), X2)
+	MOVOA(Mem{Base: R12}.Offset(32), X3)
+	MOVOA(X1, X4)
+	MOVQ(U32(20), RCX)
+}
+
+func MAINLOOP2() {
+	Label("MAINLOOP2")
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X3)
+	PXOR(X6, X3)
+	PADDL(X3, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X3, X3)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X1)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PXOR(X6, X0)
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X1)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X1, X1)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X3)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X3)
+	PADDL(X3, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X3, X3)
+	PXOR(X6, X0)
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X3)
+	PXOR(X6, X3)
+	PADDL(X3, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X3, X3)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X1)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X3, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PXOR(X6, X0)
+	PADDL(X0, X4)
+	MOVOA(X0, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(7), X4)
+	PSRLL(Imm(25), X6)
+	PXOR(X4, X1)
+	PXOR(X6, X1)
+	PADDL(X1, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(9), X5)
+	PSRLL(Imm(23), X6)
+	PXOR(X5, X2)
+	PSHUFL(Imm(0x93), X1, X1)
+	PXOR(X6, X2)
+	PADDL(X2, X4)
+	MOVOA(X2, X5)
+	MOVOA(X4, X6)
+	PSLLL(Imm(13), X4)
+	PSRLL(Imm(19), X6)
+	PXOR(X4, X3)
+	PSHUFL(Imm(0x4E), X2, X2)
+	PXOR(X6, X3)
+	SUBQ(Imm(4), RCX)
+	PADDL(X3, X5)
+	MOVOA(X1, X4)
+	MOVOA(X5, X6)
+	PSLLL(Imm(18), X5)
+	PXOR(X7, X7)
+	PSRLL(Imm(14), X6)
+	PXOR(X5, X0)
+	PSHUFL(Imm(0x39), X3, X3)
+	PXOR(X6, X0)
+	JA(LabelRef("MAINLOOP2"))
+	PADDL(Mem{Base: R12}.Offset(48), X0)
+	PADDL(Mem{Base: R12}.Offset(0), X1)
+	PADDL(Mem{Base: R12}.Offset(16), X2)
+	PADDL(Mem{Base: R12}.Offset(32), X3)
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X2, X2)
+	PSHUFL(Imm(0x39), X3, X3)
+	XORL(Mem{Base: SI}.Offset(0), ECX)
+	XORL(Mem{Base: SI}.Offset(48), R8L)
+	XORL(Mem{Base: SI}.Offset(32), R9L)
+	XORL(Mem{Base: SI}.Offset(16), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(0))
+	MOVL(R8L, Mem{Base: DI}.Offset(48))
+	MOVL(R9L, Mem{Base: DI}.Offset(32))
+	MOVL(EAX, Mem{Base: DI}.Offset(16))
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X2, X2)
+	PSHUFL(Imm(0x39), X3, X3)
+	XORL(Mem{Base: SI}.Offset(20), ECX)
+	XORL(Mem{Base: SI}.Offset(4), R8L)
+	XORL(Mem{Base: SI}.Offset(52), R9L)
+	XORL(Mem{Base: SI}.Offset(36), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(20))
+	MOVL(R8L, Mem{Base: DI}.Offset(4))
+	MOVL(R9L, Mem{Base: DI}.Offset(52))
+	MOVL(EAX, Mem{Base: DI}.Offset(36))
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	PSHUFL(Imm(0x39), X0, X0)
+	PSHUFL(Imm(0x39), X1, X1)
+	PSHUFL(Imm(0x39), X2, X2)
+	PSHUFL(Imm(0x39), X3, X3)
+	XORL(Mem{Base: SI}.Offset(40), ECX)
+	XORL(Mem{Base: SI}.Offset(24), R8L)
+	XORL(Mem{Base: SI}.Offset(8), R9L)
+	XORL(Mem{Base: SI}.Offset(56), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(40))
+	MOVL(R8L, Mem{Base: DI}.Offset(24))
+	MOVL(R9L, Mem{Base: DI}.Offset(8))
+	MOVL(EAX, Mem{Base: DI}.Offset(56))
+	MOVD(X0, ECX)
+	MOVD(X1, R8)
+	MOVD(X2, R9)
+	MOVD(X3, EAX)
+	XORL(Mem{Base: SI}.Offset(60), ECX)
+	XORL(Mem{Base: SI}.Offset(44), R8L)
+	XORL(Mem{Base: SI}.Offset(28), R9L)
+	XORL(Mem{Base: SI}.Offset(12), EAX)
+	MOVL(ECX, Mem{Base: DI}.Offset(60))
+	MOVL(R8L, Mem{Base: DI}.Offset(44))
+	MOVL(R9L, Mem{Base: DI}.Offset(28))
+	MOVL(EAX, Mem{Base: DI}.Offset(12))
+	MOVQ(Mem{Base: R12}.Offset(352), R9)
+	MOVL(Mem{Base: R12}.Offset(16), ECX)
+	MOVL(Mem{Base: R12}.Offset(36), R8L)
+	ADDQ(Imm(1), RCX)
+	SHLQ(Imm(32), R8)
+	ADDQ(R8, RCX)
+	MOVQ(RCX, R8)
+	SHRQ(Imm(32), R8)
+	MOVL(ECX, Mem{Base: R12}.Offset(16))
+	MOVL(R8L, Mem{Base: R12}.Offset(36))
+	CMPQ(R9, Imm(64))
+	JA(LabelRef("BYTESATLEAST65"))
+	JAE(LabelRef("BYTESATLEAST64"))
+	MOVQ(RDI, RSI)
+	MOVQ(RDX, RDI)
+	MOVQ(R9, RCX)
+	// Hack to get Avo to emit:
+	// 	REP; MOVSB
+	Instruction(&ir.Instruction{Opcode: "REP; MOVSB"})
+}

salsa20/salsa/hsalsa20.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // Package salsa provides low-level access to functions in the Salsa family.
-package salsa // import "golang.org/x/crypto/salsa20/salsa"
+package salsa
 
 import "math/bits"
 

salsa20/salsa20.go

@@ -19,7 +19,7 @@ This package also implements XSalsa20: a version of Salsa20 with a 24-byte
 nonce as specified in https://cr.yp.to/snuffle/xsalsa-20081128.pdf. Simply
 passing a 24-byte slice as the nonce triggers XSalsa20.
 */
-package salsa20 // import "golang.org/x/crypto/salsa20"
+package salsa20
 
 // TODO(agl): implement XORKeyStream12 and XORKeyStream8 - the reduced round variants of Salsa20.
 

scrypt/scrypt.go

@@ -5,7 +5,7 @@
 // Package scrypt implements the scrypt key derivation function as defined in
 // Colin Percival's paper "Stronger Key Derivation via Sequential Memory-Hard
 // Functions" (https://www.tarsnap.com/scrypt/scrypt.pdf).
-package scrypt // import "golang.org/x/crypto/scrypt"
+package scrypt
 
 import (
 	"crypto/sha256"

sha3/_asm/go.mod

@@ -0,0 +1,15 @@
+module sha3/_asm
+
+go 1.22
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.25.0
+)
+
+require (
+	golang.org/x/mod v0.19.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/tools v0.23.0 // indirect
+)

sha3/_asm/go.sum

@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
+golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
+golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=

sha3/_asm/keccakf_amd64_asm.go

@@ -0,0 +1,438 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources at https://github.com/gvanas/KeccakCodePackage
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/sha3"
+)
+
+//go:generate go run . -out ../keccakf_amd64.s -pkg sha3
+
+// Round Constants for use in the ι step.
+var RoundConstants = [24]uint64{
+	0x0000000000000001,
+	0x0000000000008082,
+	0x800000000000808A,
+	0x8000000080008000,
+	0x000000000000808B,
+	0x0000000080000001,
+	0x8000000080008081,
+	0x8000000000008009,
+	0x000000000000008A,
+	0x0000000000000088,
+	0x0000000080008009,
+	0x000000008000000A,
+	0x000000008000808B,
+	0x800000000000008B,
+	0x8000000000008089,
+	0x8000000000008003,
+	0x8000000000008002,
+	0x8000000000000080,
+	0x000000000000800A,
+	0x800000008000000A,
+	0x8000000080008081,
+	0x8000000000008080,
+	0x0000000080000001,
+	0x8000000080008008,
+}
+
+var (
+	// Temporary registers
+	rT1 GPPhysical = RAX
+
+	// Round vars
+	rpState = Mem{Base: RDI}
+	rpStack = Mem{Base: RSP}
+
+	rDa = RBX
+	rDe = RCX
+	rDi = RDX
+	rDo = R8
+	rDu = R9
+
+	rBa = R10
+	rBe = R11
+	rBi = R12
+	rBo = R13
+	rBu = R14
+
+	rCa = RSI
+	rCe = RBP
+	rCi = rBi
+	rCo = rBo
+	rCu = R15
+)
+
+const (
+	_ba = iota * 8
+	_be
+	_bi
+	_bo
+	_bu
+	_ga
+	_ge
+	_gi
+	_go
+	_gu
+	_ka
+	_ke
+	_ki
+	_ko
+	_ku
+	_ma
+	_me
+	_mi
+	_mo
+	_mu
+	_sa
+	_se
+	_si
+	_so
+	_su
+)
+
+func main() {
+	Package("golang.org/x/crypto/sha3")
+	ConstraintExpr("amd64,!purego,gc")
+	keccakF1600()
+	Generate()
+}
+
+func MOVQ_RBI_RCE() { MOVQ(rBi, rCe) }
+func XORQ_RT1_RCA() { XORQ(rT1, rCa) }
+func XORQ_RT1_RCE() { XORQ(rT1, rCe) }
+func XORQ_RBA_RCU() { XORQ(rBa, rCu) }
+func XORQ_RBE_RCU() { XORQ(rBe, rCu) }
+func XORQ_RDU_RCU() { XORQ(rDu, rCu) }
+func XORQ_RDA_RCA() { XORQ(rDa, rCa) }
+func XORQ_RDE_RCE() { XORQ(rDe, rCe) }
+
+type ArgMacro func()
+
+func mKeccakRound(
+	iState, oState Mem,
+	rc U64,
+	B_RBI_RCE, G_RT1_RCA, G_RT1_RCE, G_RBA_RCU,
+	K_RT1_RCA, K_RT1_RCE, K_RBA_RCU, M_RT1_RCA,
+	M_RT1_RCE, M_RBE_RCU, S_RDU_RCU, S_RDA_RCA,
+	S_RDE_RCE ArgMacro,
+) {
+	Comment("Prepare round")
+	MOVQ(rCe, rDa)
+	ROLQ(Imm(1), rDa)
+
+	MOVQ(iState.Offset(_bi), rCi)
+	XORQ(iState.Offset(_gi), rDi)
+	XORQ(rCu, rDa)
+	XORQ(iState.Offset(_ki), rCi)
+	XORQ(iState.Offset(_mi), rDi)
+	XORQ(rDi, rCi)
+
+	MOVQ(rCi, rDe)
+	ROLQ(Imm(1), rDe)
+
+	MOVQ(iState.Offset(_bo), rCo)
+	XORQ(iState.Offset(_go), rDo)
+	XORQ(rCa, rDe)
+	XORQ(iState.Offset(_ko), rCo)
+	XORQ(iState.Offset(_mo), rDo)
+	XORQ(rDo, rCo)
+
+	MOVQ(rCo, rDi)
+	ROLQ(Imm(1), rDi)
+
+	MOVQ(rCu, rDo)
+	XORQ(rCe, rDi)
+	ROLQ(Imm(1), rDo)
+
+	MOVQ(rCa, rDu)
+	XORQ(rCi, rDo)
+	ROLQ(Imm(1), rDu)
+
+	Comment("Result b")
+	MOVQ(iState.Offset(_ba), rBa)
+	MOVQ(iState.Offset(_ge), rBe)
+	XORQ(rCo, rDu)
+	MOVQ(iState.Offset(_ki), rBi)
+	MOVQ(iState.Offset(_mo), rBo)
+	MOVQ(iState.Offset(_su), rBu)
+	XORQ(rDe, rBe)
+	ROLQ(Imm(44), rBe)
+	XORQ(rDi, rBi)
+	XORQ(rDa, rBa)
+	ROLQ(Imm(43), rBi)
+
+	MOVQ(rBe, rCa)
+	MOVQ(rc, rT1)
+	ORQ(rBi, rCa)
+	XORQ(rBa, rT1)
+	XORQ(rT1, rCa)
+	MOVQ(rCa, oState.Offset(_ba))
+
+	XORQ(rDu, rBu)
+	ROLQ(Imm(14), rBu)
+	MOVQ(rBa, rCu)
+	ANDQ(rBe, rCu)
+	XORQ(rBu, rCu)
+	MOVQ(rCu, oState.Offset(_bu))
+
+	XORQ(rDo, rBo)
+	ROLQ(Imm(21), rBo)
+	MOVQ(rBo, rT1)
+	ANDQ(rBu, rT1)
+	XORQ(rBi, rT1)
+	MOVQ(rT1, oState.Offset(_bi))
+
+	NOTQ(rBi)
+	ORQ(rBa, rBu)
+	ORQ(rBo, rBi)
+	XORQ(rBo, rBu)
+	XORQ(rBe, rBi)
+	MOVQ(rBu, oState.Offset(_bo))
+	MOVQ(rBi, oState.Offset(_be))
+	B_RBI_RCE()
+
+	Comment("Result g")
+	MOVQ(iState.Offset(_gu), rBe)
+	XORQ(rDu, rBe)
+	MOVQ(iState.Offset(_ka), rBi)
+	ROLQ(Imm(20), rBe)
+	XORQ(rDa, rBi)
+	ROLQ(Imm(3), rBi)
+	MOVQ(iState.Offset(_bo), rBa)
+	MOVQ(rBe, rT1)
+	ORQ(rBi, rT1)
+	XORQ(rDo, rBa)
+	MOVQ(iState.Offset(_me), rBo)
+	MOVQ(iState.Offset(_si), rBu)
+	ROLQ(Imm(28), rBa)
+	XORQ(rBa, rT1)
+	MOVQ(rT1, oState.Offset(_ga))
+	G_RT1_RCA()
+
+	XORQ(rDe, rBo)
+	ROLQ(Imm(45), rBo)
+	MOVQ(rBi, rT1)
+	ANDQ(rBo, rT1)
+	XORQ(rBe, rT1)
+	MOVQ(rT1, oState.Offset(_ge))
+	G_RT1_RCE()
+
+	XORQ(rDi, rBu)
+	ROLQ(Imm(61), rBu)
+	MOVQ(rBu, rT1)
+	ORQ(rBa, rT1)
+	XORQ(rBo, rT1)
+	MOVQ(rT1, oState.Offset(_go))
+
+	ANDQ(rBe, rBa)
+	XORQ(rBu, rBa)
+	MOVQ(rBa, oState.Offset(_gu))
+	NOTQ(rBu)
+	G_RBA_RCU()
+
+	ORQ(rBu, rBo)
+	XORQ(rBi, rBo)
+	MOVQ(rBo, oState.Offset(_gi))
+
+	Comment("Result k")
+	MOVQ(iState.Offset(_be), rBa)
+	MOVQ(iState.Offset(_gi), rBe)
+	MOVQ(iState.Offset(_ko), rBi)
+	MOVQ(iState.Offset(_mu), rBo)
+	MOVQ(iState.Offset(_sa), rBu)
+	XORQ(rDi, rBe)
+	ROLQ(Imm(6), rBe)
+	XORQ(rDo, rBi)
+	ROLQ(Imm(25), rBi)
+	MOVQ(rBe, rT1)
+	ORQ(rBi, rT1)
+	XORQ(rDe, rBa)
+	ROLQ(Imm(1), rBa)
+	XORQ(rBa, rT1)
+	MOVQ(rT1, oState.Offset(_ka))
+	K_RT1_RCA()
+
+	XORQ(rDu, rBo)
+	ROLQ(Imm(8), rBo)
+	MOVQ(rBi, rT1)
+	ANDQ(rBo, rT1)
+	XORQ(rBe, rT1)
+	MOVQ(rT1, oState.Offset(_ke))
+	K_RT1_RCE()
+
+	XORQ(rDa, rBu)
+	ROLQ(Imm(18), rBu)
+	NOTQ(rBo)
+	MOVQ(rBo, rT1)
+	ANDQ(rBu, rT1)
+	XORQ(rBi, rT1)
+	MOVQ(rT1, oState.Offset(_ki))
+
+	MOVQ(rBu, rT1)
+	ORQ(rBa, rT1)
+	XORQ(rBo, rT1)
+	MOVQ(rT1, oState.Offset(_ko))
+
+	ANDQ(rBe, rBa)
+	XORQ(rBu, rBa)
+	MOVQ(rBa, oState.Offset(_ku))
+	K_RBA_RCU()
+
+	Comment("Result m")
+	MOVQ(iState.Offset(_ga), rBe)
+	XORQ(rDa, rBe)
+	MOVQ(iState.Offset(_ke), rBi)
+	ROLQ(Imm(36), rBe)
+	XORQ(rDe, rBi)
+	MOVQ(iState.Offset(_bu), rBa)
+	ROLQ(Imm(10), rBi)
+	MOVQ(rBe, rT1)
+	MOVQ(iState.Offset(_mi), rBo)
+	ANDQ(rBi, rT1)
+	XORQ(rDu, rBa)
+	MOVQ(iState.Offset(_so), rBu)
+	ROLQ(Imm(27), rBa)
+	XORQ(rBa, rT1)
+	MOVQ(rT1, oState.Offset(_ma))
+	M_RT1_RCA()
+
+	XORQ(rDi, rBo)
+	ROLQ(Imm(15), rBo)
+	MOVQ(rBi, rT1)
+	ORQ(rBo, rT1)
+	XORQ(rBe, rT1)
+	MOVQ(rT1, oState.Offset(_me))
+	M_RT1_RCE()
+
+	XORQ(rDo, rBu)
+	ROLQ(Imm(56), rBu)
+	NOTQ(rBo)
+	MOVQ(rBo, rT1)
+	ORQ(rBu, rT1)
+	XORQ(rBi, rT1)
+	MOVQ(rT1, oState.Offset(_mi))
+
+	ORQ(rBa, rBe)
+	XORQ(rBu, rBe)
+	MOVQ(rBe, oState.Offset(_mu))
+
+	ANDQ(rBa, rBu)
+	XORQ(rBo, rBu)
+	MOVQ(rBu, oState.Offset(_mo))
+	M_RBE_RCU()
+
+	Comment("Result s")
+	MOVQ(iState.Offset(_bi), rBa)
+	MOVQ(iState.Offset(_go), rBe)
+	MOVQ(iState.Offset(_ku), rBi)
+	XORQ(rDi, rBa)
+	MOVQ(iState.Offset(_ma), rBo)
+	ROLQ(Imm(62), rBa)
+	XORQ(rDo, rBe)
+	MOVQ(iState.Offset(_se), rBu)
+	ROLQ(Imm(55), rBe)
+
+	XORQ(rDu, rBi)
+	MOVQ(rBa, rDu)
+	XORQ(rDe, rBu)
+	ROLQ(Imm(2), rBu)
+	ANDQ(rBe, rDu)
+	XORQ(rBu, rDu)
+	MOVQ(rDu, oState.Offset(_su))
+
+	ROLQ(Imm(39), rBi)
+	S_RDU_RCU()
+	NOTQ(rBe)
+	XORQ(rDa, rBo)
+	MOVQ(rBe, rDa)
+	ANDQ(rBi, rDa)
+	XORQ(rBa, rDa)
+	MOVQ(rDa, oState.Offset(_sa))
+	S_RDA_RCA()
+
+	ROLQ(Imm(41), rBo)
+	MOVQ(rBi, rDe)
+	ORQ(rBo, rDe)
+	XORQ(rBe, rDe)
+	MOVQ(rDe, oState.Offset(_se))
+	S_RDE_RCE()
+
+	MOVQ(rBo, rDi)
+	MOVQ(rBu, rDo)
+	ANDQ(rBu, rDi)
+	ORQ(rBa, rDo)
+	XORQ(rBi, rDi)
+	XORQ(rBo, rDo)
+	MOVQ(rDi, oState.Offset(_si))
+	MOVQ(rDo, oState.Offset(_so))
+}
+
+// keccakF1600 applies the Keccak permutation to a 1600b-wide
+// state represented as a slice of 25 uint64s.
+func keccakF1600() {
+	Implement("keccakF1600")
+	AllocLocal(200)
+
+	Load(Param("a"), rpState.Base)
+
+	Comment("Convert the user state into an internal state")
+	NOTQ(rpState.Offset(_be))
+	NOTQ(rpState.Offset(_bi))
+	NOTQ(rpState.Offset(_go))
+	NOTQ(rpState.Offset(_ki))
+	NOTQ(rpState.Offset(_mi))
+	NOTQ(rpState.Offset(_sa))
+
+	Comment("Execute the KeccakF permutation")
+	MOVQ(rpState.Offset(_ba), rCa)
+	MOVQ(rpState.Offset(_be), rCe)
+	MOVQ(rpState.Offset(_bu), rCu)
+
+	XORQ(rpState.Offset(_ga), rCa)
+	XORQ(rpState.Offset(_ge), rCe)
+	XORQ(rpState.Offset(_gu), rCu)
+
+	XORQ(rpState.Offset(_ka), rCa)
+	XORQ(rpState.Offset(_ke), rCe)
+	XORQ(rpState.Offset(_ku), rCu)
+
+	XORQ(rpState.Offset(_ma), rCa)
+	XORQ(rpState.Offset(_me), rCe)
+	XORQ(rpState.Offset(_mu), rCu)
+
+	XORQ(rpState.Offset(_sa), rCa)
+	XORQ(rpState.Offset(_se), rCe)
+	MOVQ(rpState.Offset(_si), rDi)
+	MOVQ(rpState.Offset(_so), rDo)
+	XORQ(rpState.Offset(_su), rCu)
+
+	for i, rc := range RoundConstants[:len(RoundConstants)-1] {
+		var iState, oState Mem
+		if i%2 == 0 {
+			iState, oState = rpState, rpStack
+		} else {
+			iState, oState = rpStack, rpState
+		}
+		mKeccakRound(iState, oState, U64(rc), MOVQ_RBI_RCE, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBA_RCU, XORQ_RT1_RCA, XORQ_RT1_RCE, XORQ_RBE_RCU, XORQ_RDU_RCU, XORQ_RDA_RCA, XORQ_RDE_RCE)
+	}
+	mKeccakRound(rpStack, rpState, U64(RoundConstants[len(RoundConstants)-1]), NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP, NOP)
+
+	Comment("Revert the internal state to the user state")
+	NOTQ(rpState.Offset(_be))
+	NOTQ(rpState.Offset(_bi))
+	NOTQ(rpState.Offset(_go))
+	NOTQ(rpState.Offset(_ki))
+	NOTQ(rpState.Offset(_mi))
+	NOTQ(rpState.Offset(_sa))
+
+	RET()
+}

sha3/allocations_test.go

@@ -0,0 +1,61 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !noopt
+
+package sha3_test
+
+import (
+	"runtime"
+	"testing"
+
+	"golang.org/x/crypto/sha3"
+)
+
+var sink byte
+
+func TestAllocations(t *testing.T) {
+	want := 0.0
+
+	if runtime.GOARCH == "s390x" {
+		// On s390x the returned hash.Hash is conditional so it escapes.
+		want = 3.0
+	}
+
+	t.Run("New", func(t *testing.T) {
+		if allocs := testing.AllocsPerRun(10, func() {
+			h := sha3.New256()
+			b := []byte("ABC")
+			h.Write(b)
+			out := make([]byte, 0, 32)
+			out = h.Sum(out)
+			sink ^= out[0]
+		}); allocs > want {
+			t.Errorf("expected zero allocations, got %0.1f", allocs)
+		}
+	})
+	t.Run("NewShake", func(t *testing.T) {
+		if allocs := testing.AllocsPerRun(10, func() {
+			h := sha3.NewShake128()
+			b := []byte("ABC")
+			h.Write(b)
+			out := make([]byte, 0, 32)
+			out = h.Sum(out)
+			sink ^= out[0]
+			h.Read(out)
+			sink ^= out[0]
+		}); allocs > want {
+			t.Errorf("expected zero allocations, got %0.1f", allocs)
+		}
+	})
+	t.Run("Sum", func(t *testing.T) {
+		if allocs := testing.AllocsPerRun(10, func() {
+			b := []byte("ABC")
+			out := sha3.Sum256(b)
+			sink ^= out[0]
+		}); allocs > want {
+			t.Errorf("expected zero allocations, got %0.1f", allocs)
+		}
+	})
+}

sha3/doc.go

@@ -59,4 +59,4 @@
 // They produce output of the same length, with the same security strengths
 // against all attacks. This means, in particular, that SHA3-256 only has
 // 128-bit collision resistance, because its output length is 32 bytes.
-package sha3 // import "golang.org/x/crypto/sha3"
+package sha3

sha3/hashes.go

@@ -9,6 +9,7 @@ package sha3
 // bytes.
 
 import (
+	"crypto"
 	"hash"
 )
 
@@ -16,39 +17,50 @@ import (
 // Its generic security strength is 224 bits against preimage attacks,
 // and 112 bits against collision attacks.
 func New224() hash.Hash {
-	if h := new224Asm(); h != nil {
-		return h
-	}
-	return &state{rate: 144, outputLen: 28, dsbyte: 0x06}
+	return new224()
 }
 
 // New256 creates a new SHA3-256 hash.
 // Its generic security strength is 256 bits against preimage attacks,
 // and 128 bits against collision attacks.
 func New256() hash.Hash {
-	if h := new256Asm(); h != nil {
-		return h
-	}
-	return &state{rate: 136, outputLen: 32, dsbyte: 0x06}
+	return new256()
 }
 
 // New384 creates a new SHA3-384 hash.
 // Its generic security strength is 384 bits against preimage attacks,
 // and 192 bits against collision attacks.
 func New384() hash.Hash {
-	if h := new384Asm(); h != nil {
-		return h
-	}
-	return &state{rate: 104, outputLen: 48, dsbyte: 0x06}
+	return new384()
 }
 
 // New512 creates a new SHA3-512 hash.
 // Its generic security strength is 512 bits against preimage attacks,
 // and 256 bits against collision attacks.
 func New512() hash.Hash {
-	if h := new512Asm(); h != nil {
-		return h
-	}
+	return new512()
+}
+
+func init() {
+	crypto.RegisterHash(crypto.SHA3_224, New224)
+	crypto.RegisterHash(crypto.SHA3_256, New256)
+	crypto.RegisterHash(crypto.SHA3_384, New384)
+	crypto.RegisterHash(crypto.SHA3_512, New512)
+}
+
+func new224Generic() *state {
+	return &state{rate: 144, outputLen: 28, dsbyte: 0x06}
+}
+
+func new256Generic() *state {
+	return &state{rate: 136, outputLen: 32, dsbyte: 0x06}
+}
+
+func new384Generic() *state {
+	return &state{rate: 104, outputLen: 48, dsbyte: 0x06}
+}
+
+func new512Generic() *state {
 	return &state{rate: 72, outputLen: 64, dsbyte: 0x06}
 }
 

sha3/hashes_generic.go

@@ -1,27 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build !gc || purego || !s390x
-
-package sha3
-
-import (
-	"hash"
-)
-
-// new224Asm returns an assembly implementation of SHA3-224 if available,
-// otherwise it returns nil.
-func new224Asm() hash.Hash { return nil }
-
-// new256Asm returns an assembly implementation of SHA3-256 if available,
-// otherwise it returns nil.
-func new256Asm() hash.Hash { return nil }
-
-// new384Asm returns an assembly implementation of SHA3-384 if available,
-// otherwise it returns nil.
-func new384Asm() hash.Hash { return nil }
-
-// new512Asm returns an assembly implementation of SHA3-512 if available,
-// otherwise it returns nil.
-func new512Asm() hash.Hash { return nil }

sha3/hashes_noasm.go

@@ -0,0 +1,23 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !gc || purego || !s390x
+
+package sha3
+
+func new224() *state {
+	return new224Generic()
+}
+
+func new256() *state {
+	return new256Generic()
+}
+
+func new384() *state {
+	return new384Generic()
+}
+
+func new512() *state {
+	return new512Generic()
+}

sha3/register.go

@@ -1,18 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build go1.4
-
-package sha3
-
-import (
-	"crypto"
-)
-
-func init() {
-	crypto.RegisterHash(crypto.SHA3_224, New224)
-	crypto.RegisterHash(crypto.SHA3_256, New256)
-	crypto.RegisterHash(crypto.SHA3_384, New384)
-	crypto.RegisterHash(crypto.SHA3_512, New512)
-}

sha3/sha3.go

@@ -23,7 +23,6 @@ const (
 type state struct {
 	// Generic sponge components.
 	a    [25]uint64 // main state of the hash
-	buf  []byte     // points into storage
 	rate int        // the number of bytes of state to use
 
 	// dsbyte contains the "domain separation" bits and the first bit of
@@ -40,7 +39,8 @@ type state struct {
 	//      Extendable-Output Functions (May 2014)"
 	dsbyte byte
 
-	storage storageBuf
+	i, n    int // storage[i:n] is the buffer, i is only used while squeezing
+	storage [maxRate]byte
 
 	// Specific to SHA-3 and SHAKE.
 	outputLen int             // the default output size in bytes
@@ -54,24 +54,18 @@ func (d *state) BlockSize() int { return d.rate }
 func (d *state) Size() int { return d.outputLen }
 
 // Reset clears the internal state by zeroing the sponge state and
-// the byte buffer, and setting Sponge.state to absorbing.
+// the buffer indexes, and setting Sponge.state to absorbing.
 func (d *state) Reset() {
 	// Zero the permutation's state.
 	for i := range d.a {
 		d.a[i] = 0
 	}
 	d.state = spongeAbsorbing
-	d.buf = d.storage.asBytes()[:0]
+	d.i, d.n = 0, 0
 }
 
 func (d *state) clone() *state {
 	ret := *d
-	if ret.state == spongeAbsorbing {
-		ret.buf = ret.storage.asBytes()[:len(ret.buf)]
-	} else {
-		ret.buf = ret.storage.asBytes()[d.rate-cap(d.buf) : d.rate]
-	}
-
 	return &ret
 }
 
@@ -82,43 +76,40 @@ func (d *state) permute() {
 	case spongeAbsorbing:
 		// If we're absorbing, we need to xor the input into the state
 		// before applying the permutation.
-		xorIn(d, d.buf)
-		d.buf = d.storage.asBytes()[:0]
+		xorIn(d, d.storage[:d.rate])
+		d.n = 0
 		keccakF1600(&d.a)
 	case spongeSqueezing:
 		// If we're squeezing, we need to apply the permutation before
 		// copying more output.
 		keccakF1600(&d.a)
-		d.buf = d.storage.asBytes()[:d.rate]
-		copyOut(d, d.buf)
+		d.i = 0
+		copyOut(d, d.storage[:d.rate])
 	}
 }
 
 // pads appends the domain separation bits in dsbyte, applies
 // the multi-bitrate 10..1 padding rule, and permutes the state.
-func (d *state) padAndPermute(dsbyte byte) {
-	if d.buf == nil {
-		d.buf = d.storage.asBytes()[:0]
-	}
+func (d *state) padAndPermute() {
 	// Pad with this instance's domain-separator bits. We know that there's
 	// at least one byte of space in d.buf because, if it were full,
 	// permute would have been called to empty it. dsbyte also contains the
 	// first one bit for the padding. See the comment in the state struct.
-	d.buf = append(d.buf, dsbyte)
-	zerosStart := len(d.buf)
-	d.buf = d.storage.asBytes()[:d.rate]
-	for i := zerosStart; i < d.rate; i++ {
-		d.buf[i] = 0
+	d.storage[d.n] = d.dsbyte
+	d.n++
+	for d.n < d.rate {
+		d.storage[d.n] = 0
+		d.n++
 	}
 	// This adds the final one bit for the padding. Because of the way that
 	// bits are numbered from the LSB upwards, the final bit is the MSB of
 	// the last byte.
-	d.buf[d.rate-1] ^= 0x80
+	d.storage[d.rate-1] ^= 0x80
 	// Apply the permutation
 	d.permute()
 	d.state = spongeSqueezing
-	d.buf = d.storage.asBytes()[:d.rate]
-	copyOut(d, d.buf)
+	d.n = d.rate
+	copyOut(d, d.storage[:d.rate])
 }
 
 // Write absorbs more data into the hash's state. It panics if any
@@ -127,28 +118,25 @@ func (d *state) Write(p []byte) (written int, err error) {
 	if d.state != spongeAbsorbing {
 		panic("sha3: Write after Read")
 	}
-	if d.buf == nil {
-		d.buf = d.storage.asBytes()[:0]
-	}
 	written = len(p)
 
 	for len(p) > 0 {
-		if len(d.buf) == 0 && len(p) >= d.rate {
+		if d.n == 0 && len(p) >= d.rate {
 			// The fast path; absorb a full "rate" bytes of input and apply the permutation.
 			xorIn(d, p[:d.rate])
 			p = p[d.rate:]
 			keccakF1600(&d.a)
 		} else {
 			// The slow path; buffer the input until we can fill the sponge, and then xor it in.
-			todo := d.rate - len(d.buf)
+			todo := d.rate - d.n
 			if todo > len(p) {
 				todo = len(p)
 			}
-			d.buf = append(d.buf, p[:todo]...)
+			d.n += copy(d.storage[d.n:], p[:todo])
 			p = p[todo:]
 
 			// If the sponge is full, apply the permutation.
-			if len(d.buf) == d.rate {
+			if d.n == d.rate {
 				d.permute()
 			}
 		}
@@ -161,19 +149,19 @@ func (d *state) Write(p []byte) (written int, err error) {
 func (d *state) Read(out []byte) (n int, err error) {
 	// If we're still absorbing, pad and apply the permutation.
 	if d.state == spongeAbsorbing {
-		d.padAndPermute(d.dsbyte)
+		d.padAndPermute()
 	}
 
 	n = len(out)
 
 	// Now, do the squeezing.
 	for len(out) > 0 {
-		n := copy(out, d.buf)
-		d.buf = d.buf[n:]
+		n := copy(out, d.storage[d.i:d.n])
+		d.i += n
 		out = out[n:]
 
 		// Apply the permutation if we've squeezed the sponge dry.
-		if len(d.buf) == 0 {
+		if d.i == d.rate {
 			d.permute()
 		}
 	}

sha3/sha3_s390x.go

@@ -143,6 +143,12 @@ func (s *asmState) Write(b []byte) (int, error) {
 
 // Read squeezes an arbitrary number of bytes from the sponge.
 func (s *asmState) Read(out []byte) (n int, err error) {
+	// The 'compute last message digest' instruction only stores the digest
+	// at the first operand (dst) for SHAKE functions.
+	if s.function != shake_128 && s.function != shake_256 {
+		panic("sha3: can only call Read for SHAKE functions")
+	}
+
 	n = len(out)
 
 	// need to pad if we were absorbing
@@ -202,8 +208,17 @@ func (s *asmState) Sum(b []byte) []byte {
 
 	// Hash the buffer. Note that we don't clear it because we
 	// aren't updating the state.
-	klmd(s.function, &a, nil, s.buf)
-	return append(b, a[:s.outputLen]...)
+	switch s.function {
+	case sha3_224, sha3_256, sha3_384, sha3_512:
+		klmd(s.function, &a, nil, s.buf)
+		return append(b, a[:s.outputLen]...)
+	case shake_128, shake_256:
+		d := make([]byte, s.outputLen, 64)
+		klmd(s.function, &a, d, s.buf)
+		return append(b, d[:s.outputLen]...)
+	default:
+		panic("sha3: unknown function")
+	}
 }
 
 // Reset resets the Hash to its initial state.
@@ -233,56 +248,56 @@ func (s *asmState) Clone() ShakeHash {
 	return s.clone()
 }
 
-// new224Asm returns an assembly implementation of SHA3-224 if available,
-// otherwise it returns nil.
-func new224Asm() hash.Hash {
+// new224 returns an assembly implementation of SHA3-224 if available,
+// otherwise it returns a generic implementation.
+func new224() hash.Hash {
 	if cpu.S390X.HasSHA3 {
 		return newAsmState(sha3_224)
 	}
-	return nil
+	return new224Generic()
 }
 
-// new256Asm returns an assembly implementation of SHA3-256 if available,
-// otherwise it returns nil.
-func new256Asm() hash.Hash {
+// new256 returns an assembly implementation of SHA3-256 if available,
+// otherwise it returns a generic implementation.
+func new256() hash.Hash {
 	if cpu.S390X.HasSHA3 {
 		return newAsmState(sha3_256)
 	}
-	return nil
+	return new256Generic()
 }
 
-// new384Asm returns an assembly implementation of SHA3-384 if available,
-// otherwise it returns nil.
-func new384Asm() hash.Hash {
+// new384 returns an assembly implementation of SHA3-384 if available,
+// otherwise it returns a generic implementation.
+func new384() hash.Hash {
 	if cpu.S390X.HasSHA3 {
 		return newAsmState(sha3_384)
 	}
-	return nil
+	return new384Generic()
 }
 
-// new512Asm returns an assembly implementation of SHA3-512 if available,
-// otherwise it returns nil.
-func new512Asm() hash.Hash {
+// new512 returns an assembly implementation of SHA3-512 if available,
+// otherwise it returns a generic implementation.
+func new512() hash.Hash {
 	if cpu.S390X.HasSHA3 {
 		return newAsmState(sha3_512)
 	}
-	return nil
+	return new512Generic()
 }
 
-// newShake128Asm returns an assembly implementation of SHAKE-128 if available,
-// otherwise it returns nil.
-func newShake128Asm() ShakeHash {
+// newShake128 returns an assembly implementation of SHAKE-128 if available,
+// otherwise it returns a generic implementation.
+func newShake128() ShakeHash {
 	if cpu.S390X.HasSHA3 {
 		return newAsmState(shake_128)
 	}
-	return nil
+	return newShake128Generic()
 }
 
-// newShake256Asm returns an assembly implementation of SHAKE-256 if available,
-// otherwise it returns nil.
-func newShake256Asm() ShakeHash {
+// newShake256 returns an assembly implementation of SHAKE-256 if available,
+// otherwise it returns a generic implementation.
+func newShake256() ShakeHash {
 	if cpu.S390X.HasSHA3 {
 		return newAsmState(shake_256)
 	}
-	return nil
+	return newShake256Generic()
 }

sha3/sha3_test.go

@@ -76,86 +76,73 @@ type KeccakKats struct {
 	}
 }
 
-func testUnalignedAndGeneric(t *testing.T, testf func(impl string)) {
-	xorInOrig, copyOutOrig := xorIn, copyOut
-	xorIn, copyOut = xorInGeneric, copyOutGeneric
-	testf("generic")
-	if xorImplementationUnaligned != "generic" {
-		xorIn, copyOut = xorInUnaligned, copyOutUnaligned
-		testf("unaligned")
-	}
-	xorIn, copyOut = xorInOrig, copyOutOrig
-}
-
 // TestKeccakKats tests the SHA-3 and Shake implementations against all the
 // ShortMsgKATs from https://github.com/gvanas/KeccakCodePackage
 // (The testvectors are stored in keccakKats.json.deflate due to their length.)
 func TestKeccakKats(t *testing.T) {
-	testUnalignedAndGeneric(t, func(impl string) {
-		// Read the KATs.
-		deflated, err := os.Open(katFilename)
-		if err != nil {
-			t.Errorf("error opening %s: %s", katFilename, err)
-		}
-		file := flate.NewReader(deflated)
-		dec := json.NewDecoder(file)
-		var katSet KeccakKats
-		err = dec.Decode(&katSet)
-		if err != nil {
-			t.Errorf("error decoding KATs: %s", err)
-		}
+	// Read the KATs.
+	deflated, err := os.Open(katFilename)
+	if err != nil {
+		t.Errorf("error opening %s: %s", katFilename, err)
+	}
+	file := flate.NewReader(deflated)
+	dec := json.NewDecoder(file)
+	var katSet KeccakKats
+	err = dec.Decode(&katSet)
+	if err != nil {
+		t.Errorf("error decoding KATs: %s", err)
+	}
 
-		for algo, function := range testDigests {
-			d := function()
-			for _, kat := range katSet.Kats[algo] {
-				d.Reset()
-				in, err := hex.DecodeString(kat.Message)
-				if err != nil {
-					t.Errorf("error decoding KAT: %s", err)
-				}
-				d.Write(in[:kat.Length/8])
-				got := strings.ToUpper(hex.EncodeToString(d.Sum(nil)))
-				if got != kat.Digest {
-					t.Errorf("function=%s, implementation=%s, length=%d\nmessage:\n %s\ngot:\n  %s\nwanted:\n %s",
-						algo, impl, kat.Length, kat.Message, got, kat.Digest)
-					t.Logf("wanted %+v", kat)
-					t.FailNow()
-				}
-				continue
+	for algo, function := range testDigests {
+		d := function()
+		for _, kat := range katSet.Kats[algo] {
+			d.Reset()
+			in, err := hex.DecodeString(kat.Message)
+			if err != nil {
+				t.Errorf("error decoding KAT: %s", err)
 			}
-		}
-
-		for algo, v := range testShakes {
-			for _, kat := range katSet.Kats[algo] {
-				N, err := hex.DecodeString(kat.N)
-				if err != nil {
-					t.Errorf("error decoding KAT: %s", err)
-				}
-
-				S, err := hex.DecodeString(kat.S)
-				if err != nil {
-					t.Errorf("error decoding KAT: %s", err)
-				}
-				d := v.constructor(N, S)
-				in, err := hex.DecodeString(kat.Message)
-				if err != nil {
-					t.Errorf("error decoding KAT: %s", err)
-				}
-
-				d.Write(in[:kat.Length/8])
-				out := make([]byte, len(kat.Digest)/2)
-				d.Read(out)
-				got := strings.ToUpper(hex.EncodeToString(out))
-				if got != kat.Digest {
-					t.Errorf("function=%s, implementation=%s, length=%d N:%s\n S:%s\nmessage:\n %s \ngot:\n  %s\nwanted:\n %s",
-						algo, impl, kat.Length, kat.N, kat.S, kat.Message, got, kat.Digest)
-					t.Logf("wanted %+v", kat)
-					t.FailNow()
-				}
-				continue
+			d.Write(in[:kat.Length/8])
+			got := strings.ToUpper(hex.EncodeToString(d.Sum(nil)))
+			if got != kat.Digest {
+				t.Errorf("function=%s, length=%d\nmessage:\n %s\ngot:\n  %s\nwanted:\n %s",
+					algo, kat.Length, kat.Message, got, kat.Digest)
+				t.Logf("wanted %+v", kat)
+				t.FailNow()
 			}
+			continue
 		}
-	})
+	}
+
+	for algo, v := range testShakes {
+		for _, kat := range katSet.Kats[algo] {
+			N, err := hex.DecodeString(kat.N)
+			if err != nil {
+				t.Errorf("error decoding KAT: %s", err)
+			}
+
+			S, err := hex.DecodeString(kat.S)
+			if err != nil {
+				t.Errorf("error decoding KAT: %s", err)
+			}
+			d := v.constructor(N, S)
+			in, err := hex.DecodeString(kat.Message)
+			if err != nil {
+				t.Errorf("error decoding KAT: %s", err)
+			}
+
+			d.Write(in[:kat.Length/8])
+			out := make([]byte, len(kat.Digest)/2)
+			d.Read(out)
+			got := strings.ToUpper(hex.EncodeToString(out))
+			if got != kat.Digest {
+				t.Errorf("function=%s, length=%d N:%s\n S:%s\nmessage:\n %s \ngot:\n  %s\nwanted:\n %s",
+					algo, kat.Length, kat.N, kat.S, kat.Message, got, kat.Digest)
+				t.Logf("wanted %+v", kat)
+				t.FailNow()
+			}
+			continue
+		}
+	}
 }
 
 // TestKeccak does a basic test of the non-standardized Keccak hash functions.
@@ -188,122 +175,142 @@ func TestKeccak(t *testing.T) {
 	}
 }
 
+// TestShakeSum tests that the output of Sum matches the output of Read.
+func TestShakeSum(t *testing.T) {
+	tests := [...]struct {
+		name        string
+		hash        ShakeHash
+		expectedLen int
+	}{
+		{"SHAKE128", NewShake128(), 32},
+		{"SHAKE256", NewShake256(), 64},
+		{"cSHAKE128", NewCShake128([]byte{'X'}, nil), 32},
+		{"cSHAKE256", NewCShake256([]byte{'X'}, nil), 64},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			s := test.hash.Sum(nil)
+			if len(s) != test.expectedLen {
+				t.Errorf("Unexpected digest length: got %d, want %d", len(s), test.expectedLen)
+			}
+			r := make([]byte, test.expectedLen)
+			test.hash.Read(r)
+			if !bytes.Equal(s, r) {
+				t.Errorf("Mismatch between Sum and Read:\nSum:  %s\nRead: %s", hex.EncodeToString(s), hex.EncodeToString(r))
+			}
+		})
+	}
+}
+
 // TestUnalignedWrite tests that writing data in an arbitrary pattern with
 // small input buffers.
 func TestUnalignedWrite(t *testing.T) {
-	testUnalignedAndGeneric(t, func(impl string) {
-		buf := sequentialBytes(0x10000)
-		for alg, df := range testDigests {
-			d := df()
-			d.Reset()
-			d.Write(buf)
-			want := d.Sum(nil)
-			d.Reset()
-			for i := 0; i < len(buf); {
-				// Cycle through offsets which make a 137 byte sequence.
-				// Because 137 is prime this sequence should exercise all corner cases.
-				offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1}
-				for _, j := range offsets {
-					if v := len(buf) - i; v < j {
-						j = v
-					}
-					d.Write(buf[i : i+j])
-					i += j
+	buf := sequentialBytes(0x10000)
+	for alg, df := range testDigests {
+		d := df()
+		d.Reset()
+		d.Write(buf)
+		want := d.Sum(nil)
+		d.Reset()
+		for i := 0; i < len(buf); {
+			// Cycle through offsets which make a 137 byte sequence.
+			// Because 137 is prime this sequence should exercise all corner cases.
+			offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1}
+			for _, j := range offsets {
+				if v := len(buf) - i; v < j {
+					j = v
 				}
-			}
-			got := d.Sum(nil)
-			if !bytes.Equal(got, want) {
-				t.Errorf("Unaligned writes, implementation=%s, alg=%s\ngot %q, want %q", impl, alg, got, want)
+				d.Write(buf[i : i+j])
+				i += j
 			}
 		}
+		got := d.Sum(nil)
+		if !bytes.Equal(got, want) {
+			t.Errorf("Unaligned writes, alg=%s\ngot %q, want %q", alg, got, want)
+		}
+	}
 
-		// Same for SHAKE
-		for alg, df := range testShakes {
-			want := make([]byte, 16)
-			got := make([]byte, 16)
-			d := df.constructor([]byte(df.defAlgoName), []byte(df.defCustomStr))
+	// Same for SHAKE
+	for alg, df := range testShakes {
+		want := make([]byte, 16)
+		got := make([]byte, 16)
+		d := df.constructor([]byte(df.defAlgoName), []byte(df.defCustomStr))
 
-			d.Reset()
-			d.Write(buf)
-			d.Read(want)
-			d.Reset()
-			for i := 0; i < len(buf); {
-				// Cycle through offsets which make a 137 byte sequence.
-				// Because 137 is prime this sequence should exercise all corner cases.
-				offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1}
-				for _, j := range offsets {
-					if v := len(buf) - i; v < j {
-						j = v
-					}
-					d.Write(buf[i : i+j])
-					i += j
+		d.Reset()
+		d.Write(buf)
+		d.Read(want)
+		d.Reset()
+		for i := 0; i < len(buf); {
+			// Cycle through offsets which make a 137 byte sequence.
+			// Because 137 is prime this sequence should exercise all corner cases.
+			offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1}
+			for _, j := range offsets {
+				if v := len(buf) - i; v < j {
+					j = v
 				}
-			}
-			d.Read(got)
-			if !bytes.Equal(got, want) {
-				t.Errorf("Unaligned writes, implementation=%s, alg=%s\ngot %q, want %q", impl, alg, got, want)
+				d.Write(buf[i : i+j])
+				i += j
 			}
 		}
-	})
+		d.Read(got)
+		if !bytes.Equal(got, want) {
+			t.Errorf("Unaligned writes, alg=%s\ngot %q, want %q", alg, got, want)
+		}
+	}
 }
 
 // TestAppend checks that appending works when reallocation is necessary.
 func TestAppend(t *testing.T) {
-	testUnalignedAndGeneric(t, func(impl string) {
-		d := New224()
+	d := New224()
 
-		for capacity := 2; capacity <= 66; capacity += 64 {
-			// The first time around the loop, Sum will have to reallocate.
-			// The second time, it will not.
-			buf := make([]byte, 2, capacity)
-			d.Reset()
-			d.Write([]byte{0xcc})
-			buf = d.Sum(buf)
-			expected := "0000DF70ADC49B2E76EEE3A6931B93FA41841C3AF2CDF5B32A18B5478C39"
-			if got := strings.ToUpper(hex.EncodeToString(buf)); got != expected {
-				t.Errorf("got %s, want %s", got, expected)
-			}
+	for capacity := 2; capacity <= 66; capacity += 64 {
+		// The first time around the loop, Sum will have to reallocate.
+		// The second time, it will not.
+		buf := make([]byte, 2, capacity)
+		d.Reset()
+		d.Write([]byte{0xcc})
+		buf = d.Sum(buf)
+		expected := "0000DF70ADC49B2E76EEE3A6931B93FA41841C3AF2CDF5B32A18B5478C39"
+		if got := strings.ToUpper(hex.EncodeToString(buf)); got != expected {
+			t.Errorf("got %s, want %s", got, expected)
 		}
-	})
+	}
 }
 
 // TestAppendNoRealloc tests that appending works when no reallocation is necessary.
 func TestAppendNoRealloc(t *testing.T) {
-	testUnalignedAndGeneric(t, func(impl string) {
-		buf := make([]byte, 1, 200)
-		d := New224()
-		d.Write([]byte{0xcc})
-		buf = d.Sum(buf)
-		expected := "00DF70ADC49B2E76EEE3A6931B93FA41841C3AF2CDF5B32A18B5478C39"
-		if got := strings.ToUpper(hex.EncodeToString(buf)); got != expected {
-			t.Errorf("%s: got %s, want %s", impl, got, expected)
-		}
-	})
+	buf := make([]byte, 1, 200)
+	d := New224()
+	d.Write([]byte{0xcc})
+	buf = d.Sum(buf)
+	expected := "00DF70ADC49B2E76EEE3A6931B93FA41841C3AF2CDF5B32A18B5478C39"
+	if got := strings.ToUpper(hex.EncodeToString(buf)); got != expected {
+		t.Errorf("got %s, want %s", got, expected)
+	}
 }
 
 // TestSqueezing checks that squeezing the full output a single time produces
 // the same output as repeatedly squeezing the instance.
 func TestSqueezing(t *testing.T) {
-	testUnalignedAndGeneric(t, func(impl string) {
-		for algo, v := range testShakes {
-			d0 := v.constructor([]byte(v.defAlgoName), []byte(v.defCustomStr))
-			d0.Write([]byte(testString))
-			ref := make([]byte, 32)
-			d0.Read(ref)
+	for algo, v := range testShakes {
+		d0 := v.constructor([]byte(v.defAlgoName), []byte(v.defCustomStr))
+		d0.Write([]byte(testString))
+		ref := make([]byte, 32)
+		d0.Read(ref)
 
-			d1 := v.constructor([]byte(v.defAlgoName), []byte(v.defCustomStr))
-			d1.Write([]byte(testString))
-			var multiple []byte
-			for range ref {
-				one := make([]byte, 1)
-				d1.Read(one)
-				multiple = append(multiple, one...)
-			}
-			if !bytes.Equal(ref, multiple) {
-				t.Errorf("%s (%s): squeezing %d bytes one at a time failed", algo, impl, len(ref))
-			}
+		d1 := v.constructor([]byte(v.defAlgoName), []byte(v.defCustomStr))
+		d1.Write([]byte(testString))
+		var multiple []byte
+		for range ref {
+			one := make([]byte, 1)
+			d1.Read(one)
+			multiple = append(multiple, one...)
 		}
-	})
+		if !bytes.Equal(ref, multiple) {
+			t.Errorf("%s: squeezing %d bytes one at a time failed", algo, len(ref))
+		}
+	}
 }
 
 // sequentialBytes produces a buffer of size consecutive bytes 0x00, 0x01, ..., used for testing.