ssh: Make error message deterministic

By using a slice instead of a map for tried authentication methods the order is always the same. Small slices are also faster than maps.

Before the change sometimes I get the error:

    ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain

and sometimes:

    ssh: handshake failed: ssh: unable to authenticate, attempted methods [publickey none], no supported methods remain

Change-Id: I06507d57e9eef497ff05bce088d52607e69dde3e
GitHub-Last-Rev: 3a46aae4c6e3e5f52ca4b04384b3cc4efc039aa6
GitHub-Pull-Request: golang/crypto#142
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/239171
Reviewed-by: Han-Wen Nienhuys <hanwen@google.com>
Run-TryBot: Han-Wen Nienhuys <hanwen@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

.gitignore

@@ -1,2 +1,2 @@
-# Add no patterns to .hgignore except for files generated by the build.
+# Add no patterns to .gitignore except for files generated by the build.
 last-change

acme/acme.go

@@ -55,8 +55,9 @@ const (
 	ALPNProto = "acme-tls/1"
 )
 
-// idPeACMEIdentifierV1 is the OID for the ACME extension for the TLS-ALPN challenge.
-var idPeACMEIdentifierV1 = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 30, 1}
+// idPeACMEIdentifier is the OID for the ACME extension for the TLS-ALPN challenge.
+// https://tools.ietf.org/html/draft-ietf-acme-tls-alpn-05#section-5.1
+var idPeACMEIdentifier = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 31}
 
 const (
 	maxChainLen = 5       // max depth and breadth of a certificate chain
@@ -778,7 +779,7 @@ func (c *Client) TLSALPN01ChallengeCert(token, domain string, opt ...CertOption)
 		return tls.Certificate{}, err
 	}
 	acmeExtension := pkix.Extension{
-		Id:       idPeACMEIdentifierV1,
+		Id:       idPeACMEIdentifier,
 		Critical: true,
 		Value:    extValue,
 	}

acme/acme_test.go

@@ -569,6 +569,45 @@ func TestWaitAuthorization(t *testing.T) {
 			t.Errorf("err is %v (%T); want non-nil *AuthorizationError", err, err)
 		}
 	})
+	t.Run("invalid status with error returns the authorization error", func(t *testing.T) {
+		_, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprintf(w, `{
+				"type": "dns-01",
+				"status": "invalid",
+				"error": {
+				  "type": "urn:ietf:params:acme:error:caa",
+				  "detail": "CAA record for <domain> prevents issuance",
+				  "status": 403
+				},
+				"url": "https://acme-v02.api.letsencrypt.org/acme/chall-v3/xxx/xxx",
+				"token": "xxx",
+				"validationRecord": [
+				  {
+					"hostname": "<domain>"
+				  }
+				]
+			  }`)
+		})
+
+		want := &AuthorizationError{
+			Errors: []error{
+				(&wireError{
+					Status: 403,
+					Type:   "urn:ietf:params:acme:error:caa",
+					Detail: "CAA record for <domain> prevents issuance",
+				}).error(nil),
+			},
+		}
+
+		_, ok := err.(*AuthorizationError)
+		if !ok {
+			t.Errorf("err is %T; want non-nil *AuthorizationError", err)
+		}
+
+		if err.Error() != want.Error() {
+			t.Errorf("err is %v; want %v", err, want)
+		}
+	})
 	t.Run("non-retriable error", func(t *testing.T) {
 		const code = http.StatusBadRequest
 		_, err := runWaitAuthorization(context.Background(), t, func(w http.ResponseWriter, r *http.Request) {
@@ -1317,7 +1356,7 @@ func TestTLSALPN01ChallengeCert(t *testing.T) {
 	}
 	acmeExts := []pkix.Extension{}
 	for _, ext := range cert.Extensions {
-		if idPeACMEIdentifierV1.Equal(ext.Id) {
+		if idPeACMEIdentifier.Equal(ext.Id) {
 			acmeExts = append(acmeExts, ext)
 		}
 	}

acme/autocert/autocert_test.go

@@ -488,9 +488,8 @@ func startACMEServerStub(t *testing.T, tokenCert getCertificateFunc, domain stri
 		if err := crt.VerifyHostname(domain); err != nil {
 			t.Errorf("verifyTokenCert: %v", err)
 		}
-		// TODO: Update OID to the latest value 1.3.6.1.5.5.7.1.31
 		// See https://tools.ietf.org/html/draft-ietf-acme-tls-alpn-05#section-5.1
-		oid := asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 30, 1}
+		oid := asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 1, 31}
 		for _, x := range crt.Extensions {
 			if x.Id.Equal(oid) {
 				// No need to check the extension value here.

acme/autocert/listener.go

@@ -72,7 +72,6 @@ func NewListener(domains ...string) net.Listener {
 // the Manager m's Prompt, Cache, HostPolicy, and other desired options.
 func (m *Manager) Listener() net.Listener {
 	ln := &listener{
-		m:    m,
 		conf: m.TLSConfig(),
 	}
 	ln.tcpListener, ln.tcpListenErr = net.Listen("tcp", ":443")
@@ -80,7 +79,6 @@ func (m *Manager) Listener() net.Listener {
 }
 
 type listener struct {
-	m    *Manager
 	conf *tls.Config
 
 	tcpListener  net.Listener

acme/types.go

@@ -102,7 +102,12 @@ func (a *AuthorizationError) Error() string {
 	for i, err := range a.Errors {
 		e[i] = err.Error()
 	}
-	return fmt.Sprintf("acme: authorization error for %s: %s", a.Identifier, strings.Join(e, "; "))
+
+	if a.Identifier != "" {
+		return fmt.Sprintf("acme: authorization error for %s: %s", a.Identifier, strings.Join(e, "; "))
+	}
+
+	return fmt.Sprintf("acme: authorization error: %s", strings.Join(e, "; "))
 }
 
 // OrderError is returned from Client's order related methods.
@@ -407,6 +412,7 @@ type wireAuthz struct {
 	Wildcard     bool
 	Challenges   []wireChallenge
 	Combinations [][]int
+	Error        *wireError
 }
 
 func (z *wireAuthz) authorization(uri string) *Authorization {
@@ -430,11 +436,17 @@ func (z *wireAuthz) error(uri string) *AuthorizationError {
 		URI:        uri,
 		Identifier: z.Identifier.Value,
 	}
+
+	if z.Error != nil {
+		err.Errors = append(err.Errors, z.Error.error(nil))
+	}
+
 	for _, raw := range z.Challenges {
 		if raw.Error != nil {
 			err.Errors = append(err.Errors, raw.Error.error(nil))
 		}
 	}
+
 	return err
 }
 

acme/types_test.go

@@ -61,3 +61,46 @@ func TestRateLimit(t *testing.T) {
 		}
 	}
 }
+
+func TestAuthorizationError(t *testing.T) {
+	tests := []struct {
+		desc string
+		err  *AuthorizationError
+		msg  string
+	}{
+		{
+			desc: "when auth error identifier is set",
+			err: &AuthorizationError{
+				Identifier: "domain.com",
+				Errors: []error{
+					(&wireError{
+						Status: 403,
+						Type:   "urn:ietf:params:acme:error:caa",
+						Detail: "CAA record for domain.com prevents issuance",
+					}).error(nil),
+				},
+			},
+			msg: "acme: authorization error for domain.com: 403 urn:ietf:params:acme:error:caa: CAA record for domain.com prevents issuance",
+		},
+
+		{
+			desc: "when auth error identifier is unset",
+			err: &AuthorizationError{
+				Errors: []error{
+					(&wireError{
+						Status: 403,
+						Type:   "urn:ietf:params:acme:error:caa",
+						Detail: "CAA record for domain.com prevents issuance",
+					}).error(nil),
+				},
+			},
+			msg: "acme: authorization error: 403 urn:ietf:params:acme:error:caa: CAA record for domain.com prevents issuance",
+		},
+	}
+
+	for _, tt := range tests {
+		if tt.err.Error() != tt.msg {
+			t.Errorf("got: %s\nwant: %s", tt.err, tt.msg)
+		}
+	}
+}

blake2b/blake2b.go

@@ -5,6 +5,8 @@
 // Package blake2b implements the BLAKE2b hash algorithm defined by RFC 7693
 // and the extendable output function (XOF) BLAKE2Xb.
 //
+// BLAKE2b is optimized for 64-bit platforms—including NEON-enabled ARMs—and
+// produces digests of any size between 1 and 64 bytes.
 // For a detailed specification of BLAKE2b see https://blake2.net/blake2.pdf
 // and for BLAKE2Xb see https://blake2.net/blake2x.pdf
 //

blake2s/blake2s.go

@@ -5,6 +5,8 @@
 // Package blake2s implements the BLAKE2s hash algorithm defined by RFC 7693
 // and the extendable output function (XOF) BLAKE2Xs.
 //
+// BLAKE2s is optimized for 8- to 32-bit platforms and produces digests of any
+// size between 1 and 32 bytes.
 // For a detailed specification of BLAKE2s see https://blake2.net/blake2.pdf
 // and for BLAKE2Xs see https://blake2.net/blake2x.pdf
 //

chacha20/chacha_arm64.go

@@ -2,8 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.11
-// +build !gccgo,!appengine
+// +build go1.11,!gccgo,!purego
 
 package chacha20
 

chacha20/chacha_arm64.s

@@ -2,8 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.11
-// +build !gccgo,!appengine
+// +build go1.11,!gccgo,!purego
 
 #include "textflag.h"
 

chacha20/chacha_generic.go

@@ -42,10 +42,14 @@ type Cipher struct {
 
 	// The last len bytes of buf are leftover key stream bytes from the previous
 	// XORKeyStream invocation. The size of buf depends on how many blocks are
-	// computed at a time.
+	// computed at a time by xorKeyStreamBlocks.
 	buf [bufSize]byte
 	len int
 
+	// overflow is set when the counter overflowed, no more blocks can be
+	// generated, and the next XORKeyStream call should panic.
+	overflow bool
+
 	// The counter-independent results of the first round are cached after they
 	// are computed the first time.
 	precompDone      bool
@@ -89,6 +93,7 @@ func newUnauthenticatedCipher(c *Cipher, key, nonce []byte) (*Cipher, error) {
 		return nil, errors.New("chacha20: wrong nonce size")
 	}
 
+	key, nonce = key[:KeySize], nonce[:NonceSize] // bounds check elimination hint
 	c.key = [8]uint32{
 		binary.LittleEndian.Uint32(key[0:4]),
 		binary.LittleEndian.Uint32(key[4:8]),
@@ -136,6 +141,36 @@ func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) {
 	return a, b, c, d
 }
 
+// SetCounter sets the Cipher counter. The next invocation of XORKeyStream will
+// behave as if (64 * counter) bytes had been encrypted so far.
+//
+// To prevent accidental counter reuse, SetCounter panics if counter is less
+// than the current value.
+//
+// Note that the execution time of XORKeyStream is not independent of the
+// counter value.
+func (s *Cipher) SetCounter(counter uint32) {
+	// Internally, s may buffer multiple blocks, which complicates this
+	// implementation slightly. When checking whether the counter has rolled
+	// back, we must use both s.counter and s.len to determine how many blocks
+	// we have already output.
+	outputCounter := s.counter - uint32(s.len)/blockSize
+	if s.overflow || counter < outputCounter {
+		panic("chacha20: SetCounter attempted to rollback counter")
+	}
+
+	// In the general case, we set the new counter value and reset s.len to 0,
+	// causing the next call to XORKeyStream to refill the buffer. However, if
+	// we're advancing within the existing buffer, we can save work by simply
+	// setting s.len.
+	if counter < s.counter {
+		s.len = int(s.counter-counter) * blockSize
+	} else {
+		s.counter = counter
+		s.len = 0
+	}
+}
+
 // XORKeyStream XORs each byte in the given slice with a byte from the
 // cipher's key stream. Dst and src must overlap entirely or not at all.
 //
@@ -169,34 +204,52 @@ func (s *Cipher) XORKeyStream(dst, src []byte) {
 			dst[i] = src[i] ^ b
 		}
 		s.len -= len(keyStream)
-		src = src[len(keyStream):]
-		dst = dst[len(keyStream):]
+		dst, src = dst[len(keyStream):], src[len(keyStream):]
+	}
+	if len(src) == 0 {
+		return
 	}
 
-	const blocksPerBuf = bufSize / blockSize
-	numBufs := (uint64(len(src)) + bufSize - 1) / bufSize
-	if uint64(s.counter)+numBufs*blocksPerBuf >= 1<<32 {
+	// If we'd need to let the counter overflow and keep generating output,
+	// panic immediately. If instead we'd only reach the last block, remember
+	// not to generate any more output after the buffer is drained.
+	numBlocks := (uint64(len(src)) + blockSize - 1) / blockSize
+	if s.overflow || uint64(s.counter)+numBlocks > 1<<32 {
 		panic("chacha20: counter overflow")
+	} else if uint64(s.counter)+numBlocks == 1<<32 {
+		s.overflow = true
 	}
 
 	// xorKeyStreamBlocks implementations expect input lengths that are a
 	// multiple of bufSize. Platform-specific ones process multiple blocks at a
 	// time, so have bufSizes that are a multiple of blockSize.
 
-	rem := len(src) % bufSize
-	full := len(src) - rem
-
+	full := len(src) - len(src)%bufSize
 	if full > 0 {
 		s.xorKeyStreamBlocks(dst[:full], src[:full])
 	}
+	dst, src = dst[full:], src[full:]
+
+	// If using a multi-block xorKeyStreamBlocks would overflow, use the generic
+	// one that does one block at a time.
+	const blocksPerBuf = bufSize / blockSize
+	if uint64(s.counter)+blocksPerBuf > 1<<32 {
+		s.buf = [bufSize]byte{}
+		numBlocks := (len(src) + blockSize - 1) / blockSize
+		buf := s.buf[bufSize-numBlocks*blockSize:]
+		copy(buf, src)
+		s.xorKeyStreamBlocksGeneric(buf, buf)
+		s.len = len(buf) - copy(dst, buf)
+		return
+	}
 
 	// If we have a partial (multi-)block, pad it for xorKeyStreamBlocks, and
 	// keep the leftover keystream for the next XORKeyStream invocation.
-	if rem > 0 {
+	if len(src) > 0 {
 		s.buf = [bufSize]byte{}
-		copy(s.buf[:], src[full:])
+		copy(s.buf[:], src)
 		s.xorKeyStreamBlocks(s.buf[:], s.buf[:])
-		s.len = bufSize - copy(dst[full:], s.buf[:])
+		s.len = bufSize - copy(dst, s.buf[:])
 	}
 }
 
@@ -233,7 +286,9 @@ func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) {
 		s.precompDone = true
 	}
 
-	for i := 0; i < len(src); i += blockSize {
+	// A condition of len(src) > 0 would be sufficient, but this also
+	// acts as a bounds check elimination hint.
+	for len(src) >= 64 && len(dst) >= 64 {
 		// The remainder of the first column round.
 		fcr0, fcr4, fcr8, fcr12 := quarterRound(c0, c4, c8, s.counter)
 
@@ -258,49 +313,28 @@ func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) {
 			x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
 		}
 
-		// Finally, add back the initial state to generate the key stream.
-		x0 += c0
-		x1 += c1
-		x2 += c2
-		x3 += c3
-		x4 += c4
-		x5 += c5
-		x6 += c6
-		x7 += c7
-		x8 += c8
-		x9 += c9
-		x10 += c10
-		x11 += c11
-		x12 += s.counter
-		x13 += c13
-		x14 += c14
-		x15 += c15
+		// Add back the initial state to generate the key stream, then
+		// XOR the key stream with the source and write out the result.
+		addXor(dst[0:4], src[0:4], x0, c0)
+		addXor(dst[4:8], src[4:8], x1, c1)
+		addXor(dst[8:12], src[8:12], x2, c2)
+		addXor(dst[12:16], src[12:16], x3, c3)
+		addXor(dst[16:20], src[16:20], x4, c4)
+		addXor(dst[20:24], src[20:24], x5, c5)
+		addXor(dst[24:28], src[24:28], x6, c6)
+		addXor(dst[28:32], src[28:32], x7, c7)
+		addXor(dst[32:36], src[32:36], x8, c8)
+		addXor(dst[36:40], src[36:40], x9, c9)
+		addXor(dst[40:44], src[40:44], x10, c10)
+		addXor(dst[44:48], src[44:48], x11, c11)
+		addXor(dst[48:52], src[48:52], x12, s.counter)
+		addXor(dst[52:56], src[52:56], x13, c13)
+		addXor(dst[56:60], src[56:60], x14, c14)
+		addXor(dst[60:64], src[60:64], x15, c15)
 
 		s.counter += 1
-		if s.counter == 0 {
-			panic("chacha20: internal error: counter overflow")
-		}
 
-		in, out := src[i:], dst[i:]
-		in, out = in[:blockSize], out[:blockSize] // bounds check elimination hint
-
-		// XOR the key stream with the source and write out the result.
-		xor(out[0:], in[0:], x0)
-		xor(out[4:], in[4:], x1)
-		xor(out[8:], in[8:], x2)
-		xor(out[12:], in[12:], x3)
-		xor(out[16:], in[16:], x4)
-		xor(out[20:], in[20:], x5)
-		xor(out[24:], in[24:], x6)
-		xor(out[28:], in[28:], x7)
-		xor(out[32:], in[32:], x8)
-		xor(out[36:], in[36:], x9)
-		xor(out[40:], in[40:], x10)
-		xor(out[44:], in[44:], x11)
-		xor(out[48:], in[48:], x12)
-		xor(out[52:], in[52:], x13)
-		xor(out[56:], in[56:], x14)
-		xor(out[60:], in[60:], x15)
+		src, dst = src[blockSize:], dst[blockSize:]
 	}
 }
 

chacha20/chacha_noasm.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !arm64,!s390x,!ppc64le arm64,!go1.11 gccgo appengine
+// +build !arm64,!s390x,!ppc64le arm64,!go1.11 gccgo purego
 
 package chacha20
 

chacha20/chacha_ppc64le.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo,!appengine
+// +build !gccgo,!purego
 
 package chacha20
 

chacha20/chacha_ppc64le.s

@@ -19,7 +19,7 @@
 // The differences in this and the original implementation are
 // due to the calling conventions and initialization of constants.
 
-// +build !gccgo,!appengine
+// +build !gccgo,!purego
 
 #include "textflag.h"
 

chacha20/chacha_s390x.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo,!appengine
+// +build !gccgo,!purego
 
 package chacha20
 

chacha20/chacha_s390x.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo,!appengine
+// +build !gccgo,!purego
 
 #include "go_asm.h"
 #include "textflag.h"

chacha20/chacha_test.go

@@ -110,6 +110,107 @@ func TestStep(t *testing.T) {
 	}
 }
 
+func TestSetCounter(t *testing.T) {
+	newCipher := func() *Cipher {
+		s, _ := NewUnauthenticatedCipher(make([]byte, KeySize), make([]byte, NonceSize))
+		return s
+	}
+	s := newCipher()
+	src := bytes.Repeat([]byte("test"), 32) // two 64-byte blocks
+	dst1 := make([]byte, len(src))
+	s.XORKeyStream(dst1, src)
+	// advance counter to 1 and xor second block
+	s = newCipher()
+	s.SetCounter(1)
+	dst2 := make([]byte, len(src))
+	s.XORKeyStream(dst2[64:], src[64:])
+	if !bytes.Equal(dst1[64:], dst2[64:]) {
+		t.Error("failed to produce identical output using SetCounter")
+	}
+
+	// test again with unaligned blocks; SetCounter should reset the buffer
+	s = newCipher()
+	s.XORKeyStream(dst1[:70], src[:70])
+	s = newCipher()
+	s.XORKeyStream([]byte{0}, []byte{0})
+	s.SetCounter(1)
+	s.XORKeyStream(dst2[64:70], src[64:70])
+	if !bytes.Equal(dst1[64:70], dst2[64:70]) {
+		t.Error("SetCounter did not reset buffer")
+	}
+
+	// advancing to a lower counter value should cause a panic
+	panics := func(fn func()) (p bool) {
+		defer func() { p = recover() != nil }()
+		fn()
+		return
+	}
+	if !panics(func() { s.SetCounter(0) }) {
+		t.Error("counter decreasing should trigger a panic")
+	}
+}
+
+func TestLastBlock(t *testing.T) {
+	panics := func(fn func()) (p bool) {
+		defer func() { p = recover() != nil }()
+		fn()
+		return
+	}
+
+	checkLastBlock := func(b []byte) {
+		t.Helper()
+		// Hardcoded result to check all implementations generate the same output.
+		lastBlock := "ace4cd09e294d1912d4ad205d06f95d9c2f2bfcf453e8753f128765b62215f4d" +
+			"92c74f2f626c6a640c0b1284d839ec81f1696281dafc3e684593937023b58b1d"
+		if got := hex.EncodeToString(b); got != lastBlock {
+			t.Errorf("wrong output for the last block, got %q, want %q", got, lastBlock)
+		}
+	}
+
+	// setting the counter to 0xffffffff and crypting multiple blocks should
+	// trigger a panic
+	s, _ := NewUnauthenticatedCipher(make([]byte, KeySize), make([]byte, NonceSize))
+	s.SetCounter(0xffffffff)
+	blocks := make([]byte, blockSize*2)
+	if !panics(func() { s.XORKeyStream(blocks, blocks) }) {
+		t.Error("crypting multiple blocks should trigger a panic")
+	}
+
+	// setting the counter to 0xffffffff - 1 and crypting two blocks should not
+	// trigger a panic
+	s, _ = NewUnauthenticatedCipher(make([]byte, KeySize), make([]byte, NonceSize))
+	s.SetCounter(0xffffffff - 1)
+	if panics(func() { s.XORKeyStream(blocks, blocks) }) {
+		t.Error("crypting the last blocks should not trigger a panic")
+	}
+	checkLastBlock(blocks[blockSize:])
+	// once all the keystream is spent, setting the counter should panic
+	if !panics(func() { s.SetCounter(0xffffffff) }) {
+		t.Error("setting the counter after overflow should trigger a panic")
+	}
+	// crypting a subsequent block *should* panic
+	block := make([]byte, blockSize)
+	if !panics(func() { s.XORKeyStream(block, block) }) {
+		t.Error("crypting after overflow should trigger a panic")
+	}
+
+	// if we crypt less than a full block, we should be able to crypt the rest
+	// in a subsequent call without panicking
+	s, _ = NewUnauthenticatedCipher(make([]byte, KeySize), make([]byte, NonceSize))
+	s.SetCounter(0xffffffff)
+	if panics(func() { s.XORKeyStream(block[:7], block[:7]) }) {
+		t.Error("crypting part of the last block should not trigger a panic")
+	}
+	if panics(func() { s.XORKeyStream(block[7:], block[7:]) }) {
+		t.Error("crypting part of the last block should not trigger a panic")
+	}
+	checkLastBlock(block)
+	// as before, a third call should trigger a panic because all keystream is spent
+	if !panics(func() { s.XORKeyStream(block[:1], block[:1]) }) {
+		t.Error("crypting after overflow should trigger a panic")
+	}
+}
+
 func benchmarkChaCha20(b *testing.B, step, count int) {
 	tot := step * count
 	src := make([]byte, tot)

chacha20/xor.go

@@ -13,10 +13,10 @@ const unaligned = runtime.GOARCH == "386" ||
 	runtime.GOARCH == "ppc64le" ||
 	runtime.GOARCH == "s390x"
 
-// xor reads a little endian uint32 from src, XORs it with u and
+// addXor reads a little endian uint32 from src, XORs it with (a + b) and
 // places the result in little endian byte order in dst.
-func xor(dst, src []byte, u uint32) {
-	_, _ = src[3], dst[3] // eliminate bounds checks
+func addXor(dst, src []byte, a, b uint32) {
+	_, _ = src[3], dst[3] // bounds check elimination hint
 	if unaligned {
 		// The compiler should optimize this code into
 		// 32-bit unaligned little endian loads and stores.
@@ -27,15 +27,16 @@ func xor(dst, src []byte, u uint32) {
 		v |= uint32(src[1]) << 8
 		v |= uint32(src[2]) << 16
 		v |= uint32(src[3]) << 24
-		v ^= u
+		v ^= a + b
 		dst[0] = byte(v)
 		dst[1] = byte(v >> 8)
 		dst[2] = byte(v >> 16)
 		dst[3] = byte(v >> 24)
 	} else {
-		dst[0] = src[0] ^ byte(u)
-		dst[1] = src[1] ^ byte(u>>8)
-		dst[2] = src[2] ^ byte(u>>16)
-		dst[3] = src[3] ^ byte(u>>24)
+		a += b
+		dst[0] = src[0] ^ byte(a)
+		dst[1] = src[1] ^ byte(a>>8)
+		dst[2] = src[2] ^ byte(a>>16)
+		dst[3] = src[3] ^ byte(a>>24)
 	}
 }

chacha20poly1305/chacha20poly1305_amd64.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.7,amd64,!gccgo,!appengine
+// +build !gccgo,!purego
 
 package chacha20poly1305
 

chacha20poly1305/chacha20poly1305_amd64.s

@@ -4,7 +4,7 @@
 
 // This file was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare.
 
-// +build go1.7,amd64,!gccgo,!appengine
+// +build !gccgo,!purego
 
 #include "textflag.h"
 // General register allocation

chacha20poly1305/chacha20poly1305_generic.go

@@ -12,56 +12,64 @@ import (
 	"golang.org/x/crypto/poly1305"
 )
 
-func roundTo16(n int) int {
-	return 16 * ((n + 15) / 16)
+func writeWithPadding(p *poly1305.MAC, b []byte) {
+	p.Write(b)
+	if rem := len(b) % 16; rem != 0 {
+		var buf [16]byte
+		padLen := 16 - rem
+		p.Write(buf[:padLen])
+	}
+}
+
+func writeUint64(p *poly1305.MAC, n int) {
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], uint64(n))
+	p.Write(buf[:])
 }
 
 func (c *chacha20poly1305) sealGeneric(dst, nonce, plaintext, additionalData []byte) []byte {
 	ret, out := sliceForAppend(dst, len(plaintext)+poly1305.TagSize)
+	ciphertext, tag := out[:len(plaintext)], out[len(plaintext):]
 	if subtle.InexactOverlap(out, plaintext) {
 		panic("chacha20poly1305: invalid buffer overlap")
 	}
 
-	var polyKey, discardBuf [32]byte
+	var polyKey [32]byte
 	s, _ := chacha20.NewUnauthenticatedCipher(c.key[:], nonce)
 	s.XORKeyStream(polyKey[:], polyKey[:])
-	s.XORKeyStream(discardBuf[:], discardBuf[:]) // skip the next 32 bytes
-	s.XORKeyStream(out, plaintext)
+	s.SetCounter(1) // set the counter to 1, skipping 32 bytes
+	s.XORKeyStream(ciphertext, plaintext)
 
-	polyInput := make([]byte, roundTo16(len(additionalData))+roundTo16(len(plaintext))+8+8)
-	copy(polyInput, additionalData)
-	copy(polyInput[roundTo16(len(additionalData)):], out[:len(plaintext)])
-	binary.LittleEndian.PutUint64(polyInput[len(polyInput)-16:], uint64(len(additionalData)))
-	binary.LittleEndian.PutUint64(polyInput[len(polyInput)-8:], uint64(len(plaintext)))
-
-	var tag [poly1305.TagSize]byte
-	poly1305.Sum(&tag, polyInput, &polyKey)
-	copy(out[len(plaintext):], tag[:])
+	p := poly1305.New(&polyKey)
+	writeWithPadding(p, additionalData)
+	writeWithPadding(p, ciphertext)
+	writeUint64(p, len(additionalData))
+	writeUint64(p, len(plaintext))
+	p.Sum(tag[:0])
 
 	return ret
 }
 
 func (c *chacha20poly1305) openGeneric(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
-	var tag [poly1305.TagSize]byte
-	copy(tag[:], ciphertext[len(ciphertext)-16:])
+	tag := ciphertext[len(ciphertext)-16:]
 	ciphertext = ciphertext[:len(ciphertext)-16]
 
-	var polyKey, discardBuf [32]byte
+	var polyKey [32]byte
 	s, _ := chacha20.NewUnauthenticatedCipher(c.key[:], nonce)
 	s.XORKeyStream(polyKey[:], polyKey[:])
-	s.XORKeyStream(discardBuf[:], discardBuf[:]) // skip the next 32 bytes
+	s.SetCounter(1) // set the counter to 1, skipping 32 bytes
 
-	polyInput := make([]byte, roundTo16(len(additionalData))+roundTo16(len(ciphertext))+8+8)
-	copy(polyInput, additionalData)
-	copy(polyInput[roundTo16(len(additionalData)):], ciphertext)
-	binary.LittleEndian.PutUint64(polyInput[len(polyInput)-16:], uint64(len(additionalData)))
-	binary.LittleEndian.PutUint64(polyInput[len(polyInput)-8:], uint64(len(ciphertext)))
+	p := poly1305.New(&polyKey)
+	writeWithPadding(p, additionalData)
+	writeWithPadding(p, ciphertext)
+	writeUint64(p, len(additionalData))
+	writeUint64(p, len(ciphertext))
 
 	ret, out := sliceForAppend(dst, len(ciphertext))
 	if subtle.InexactOverlap(out, ciphertext) {
 		panic("chacha20poly1305: invalid buffer overlap")
 	}
-	if !poly1305.Verify(&tag, polyInput, &polyKey) {
+	if !p.Verify(tag) {
 		for i := range out {
 			out[i] = 0
 		}

chacha20poly1305/chacha20poly1305_noasm.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !amd64 !go1.7 gccgo appengine
+// +build !amd64 gccgo purego
 
 package chacha20poly1305
 

chacha20poly1305/chacha20poly1305_test.go

@@ -10,7 +10,6 @@ import (
 	cryptorand "crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"log"
 	mathrand "math/rand"
 	"strconv"
 	"testing"
@@ -220,36 +219,50 @@ func BenchmarkChacha20Poly1305(b *testing.B) {
 	}
 }
 
-var key = make([]byte, KeySize)
-
 func ExampleNewX() {
-	aead, err := NewX(key)
-	if err != nil {
-		log.Fatalln("Failed to instantiate XChaCha20-Poly1305:", err)
+	// key should be randomly generated or derived from a function like Argon2.
+	key := make([]byte, KeySize)
+	if _, err := cryptorand.Read(key); err != nil {
+		panic(err)
 	}
 
-	for _, msg := range []string{
-		"Attack at dawn.",
-		"The eagle has landed.",
-		"Gophers, gophers, gophers everywhere!",
-	} {
-		// Encryption.
-		nonce := make([]byte, NonceSizeX)
+	aead, err := NewX(key)
+	if err != nil {
+		panic(err)
+	}
+
+	// Encryption.
+	var encryptedMsg []byte
+	{
+		msg := []byte("Gophers, gophers, gophers everywhere!")
+
+		// Select a random nonce, and leave capacity for the ciphertext.
+		nonce := make([]byte, aead.NonceSize(), aead.NonceSize()+len(msg)+aead.Overhead())
 		if _, err := cryptorand.Read(nonce); err != nil {
 			panic(err)
 		}
-		ciphertext := aead.Seal(nil, nonce, []byte(msg), nil)
 
-		// Decryption.
+		// Encrypt the message and append the ciphertext to the nonce.
+		encryptedMsg = aead.Seal(nonce, nonce, msg, nil)
+	}
+
+	// Decryption.
+	{
+		if len(encryptedMsg) < aead.NonceSize() {
+			panic("ciphertext too short")
+		}
+
+		// Split nonce and ciphertext.
+		nonce, ciphertext := encryptedMsg[:aead.NonceSize()], encryptedMsg[aead.NonceSize():]
+
+		// Decrypt the message and check it wasn't tampered with.
 		plaintext, err := aead.Open(nil, nonce, ciphertext, nil)
 		if err != nil {
-			log.Fatalln("Failed to decrypt or authenticate message:", err)
+			panic(err)
 		}
 
 		fmt.Printf("%s\n", plaintext)
 	}
 
-	// Output: Attack at dawn.
-	// The eagle has landed.
-	// Gophers, gophers, gophers everywhere!
+	// Output: Gophers, gophers, gophers everywhere!
 }

cryptobyte/asn1.go

@@ -81,7 +81,7 @@ func (b *Builder) AddASN1BigInt(n *big.Int) {
 			for i := range bytes {
 				bytes[i] ^= 0xff
 			}
-			if bytes[0]&0x80 == 0 {
+			if len(bytes) == 0 || bytes[0]&0x80 == 0 {
 				c.add(0xff)
 			}
 			c.add(bytes...)
@@ -230,12 +230,12 @@ func (b *Builder) AddASN1(tag asn1.Tag, f BuilderContinuation) {
 
 // String
 
-// ReadASN1Boolean decodes an ASN.1 INTEGER and converts it to a boolean
+// ReadASN1Boolean decodes an ASN.1 BOOLEAN and converts it to a boolean
 // representation into out and advances. It reports whether the read
 // was successful.
 func (s *String) ReadASN1Boolean(out *bool) bool {
 	var bytes String
-	if !s.ReadASN1(&bytes, asn1.INTEGER) || len(bytes) != 1 {
+	if !s.ReadASN1(&bytes, asn1.BOOLEAN) || len(bytes) != 1 {
 		return false
 	}
 

cryptobyte/asn1_test.go

@@ -335,3 +335,40 @@ func TestReadASN1BitString(t *testing.T) {
 		}
 	}
 }
+
+func TestAddASN1BigInt(t *testing.T) {
+	x := big.NewInt(-1)
+	var b Builder
+	b.AddASN1BigInt(x)
+	got, err := b.Bytes()
+	if err != nil {
+		t.Fatalf("unexpected error adding -1: %v", err)
+	}
+	s := String(got)
+	var y big.Int
+	ok := s.ReadASN1Integer(&y)
+	if !ok || x.Cmp(&y) != 0 {
+		t.Errorf("unexpected bytes %v, want %v", &y, x)
+	}
+}
+
+func TestReadASN1Boolean(t *testing.T) {
+	testData := []struct {
+		in  []byte
+		ok  bool
+		out bool
+	}{
+		{[]byte{}, false, false},
+		{[]byte{0x01, 0x01, 0x00}, true, false},
+		{[]byte{0x01, 0x01, 0xff}, true, true},
+		{[]byte{0x01, 0x01, 0x01}, false, false},
+	}
+	for i, test := range testData {
+		in := String(test.in)
+		var out bool
+		ok := in.ReadASN1Boolean(&out)
+		if ok != test.ok || ok && (out != test.out) {
+			t.Errorf("#%d: in.ReadASN1Boolean() = %v, want %v; out = %v, want %v", i, ok, test.ok, out, test.out)
+		}
+	}
+}

internal/wycheproof/aes_cbc_test.go

@@ -0,0 +1,127 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wycheproof
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/hex"
+	"fmt"
+	"testing"
+)
+
+func TestAesCbc(t *testing.T) {
+	// IndCpaTestVector
+	type IndCpaTestVector struct {
+
+		// A brief description of the test case
+		Comment string `json:"comment,omitempty"`
+
+		// the raw ciphertext (without IV)
+		Ct string `json:"ct,omitempty"`
+
+		// A list of flags
+		Flags []string `json:"flags,omitempty"`
+
+		// the initialization vector
+		Iv string `json:"iv,omitempty"`
+
+		// the key
+		Key string `json:"key,omitempty"`
+
+		// the plaintext
+		Msg string `json:"msg,omitempty"`
+
+		// Test result
+		Result string `json:"result,omitempty"`
+
+		// Identifier of the test case
+		TcId int `json:"tcId,omitempty"`
+	}
+
+	// Notes a description of the labels used in the test vectors
+	type Notes struct {
+	}
+
+	// IndCpaTestGroup
+	type IndCpaTestGroup struct {
+
+		// the IV size in bits
+		IvSize int `json:"ivSize,omitempty"`
+
+		// the keySize in bits
+		KeySize int `json:"keySize,omitempty"`
+
+		// the expected size of the tag in bits
+		TagSize int                 `json:"tagSize,omitempty"`
+		Tests   []*IndCpaTestVector `json:"tests,omitempty"`
+		Type    interface{}         `json:"type,omitempty"`
+	}
+
+	// Root
+	type Root struct {
+
+		// the primitive tested in the test file
+		Algorithm string `json:"algorithm,omitempty"`
+
+		// the version of the test vectors.
+		GeneratorVersion string `json:"generatorVersion,omitempty"`
+
+		// additional documentation
+		Header []string `json:"header,omitempty"`
+
+		// a description of the labels used in the test vectors
+		Notes *Notes `json:"notes,omitempty"`
+
+		// the number of test vectors in this test
+		NumberOfTests int                `json:"numberOfTests,omitempty"`
+		Schema        interface{}        `json:"schema,omitempty"`
+		TestGroups    []*IndCpaTestGroup `json:"testGroups,omitempty"`
+	}
+
+	var root Root
+	readTestVector(t, "aes_cbc_pkcs5_test.json", &root)
+	for _, tg := range root.TestGroups {
+	tests:
+		for _, tv := range tg.Tests {
+			block, err := aes.NewCipher(decodeHex(tv.Key))
+			if err != nil {
+				t.Fatalf("#%d: %v", tv.TcId, err)
+			}
+			mode := cipher.NewCBCDecrypter(block, decodeHex(tv.Iv))
+			ct := decodeHex(tv.Ct)
+			if len(ct)%aes.BlockSize != 0 {
+				panic(fmt.Sprintf("#%d: ciphertext is not a multiple of the block size", tv.TcId))
+			}
+			mode.CryptBlocks(ct, ct) // decrypt the block in place
+
+			// Skip the tests that are broken due to bad padding. Panic if there are any
+			// tests left that are invalid for some other reason in the future, to
+			// evaluate what to do with those tests.
+			for _, flag := range tv.Flags {
+				if flag == "BadPadding" {
+					continue tests
+				}
+			}
+			if !shouldPass(tv.Result, tv.Flags, nil) {
+				panic(fmt.Sprintf("#%d: found an invalid test that is broken for some reason other than bad padding", tv.TcId))
+			}
+
+			// Remove the PKCS#5 padding from the given ciphertext to validate it
+			padding := ct[len(ct)-1]
+			paddingNum := int(padding)
+			for i := paddingNum; i > 0; i-- {
+				if ct[len(ct)-i] != padding { // panic if the padding is unexpectedly bad
+					panic(fmt.Sprintf("#%d: bad padding at index=%d of %v", tv.TcId, i, ct))
+				}
+			}
+			ct = ct[:len(ct)-paddingNum]
+
+			if got, want := hex.EncodeToString(ct), tv.Msg; got != want {
+				t.Errorf("#%d, type: %s, comment: %q, decoded ciphertext not equal: %s, want %s", tv.TcId, tv.Result, tv.Comment, got, want)
+			}
+		}
+	}
+}

internal/wycheproof/chacha20_poly1305_test.go

@@ -0,0 +1,148 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wycheproof
+
+import (
+	"crypto/cipher"
+	"encoding/hex"
+	"testing"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+func TestChaCha20Poly1305(t *testing.T) {
+	// AeadTestVector
+	type AeadTestVector struct {
+
+		// additional authenticated data
+		Aad string `json:"aad,omitempty"`
+
+		// A brief description of the test case
+		Comment string `json:"comment,omitempty"`
+
+		// the ciphertext (without iv and tag)
+		Ct string `json:"ct,omitempty"`
+
+		// A list of flags
+		Flags []string `json:"flags,omitempty"`
+
+		// the nonce
+		Iv string `json:"iv,omitempty"`
+
+		// the key
+		Key string `json:"key,omitempty"`
+
+		// the plaintext
+		Msg string `json:"msg,omitempty"`
+
+		// Test result
+		Result string `json:"result,omitempty"`
+
+		// the authentication tag
+		Tag string `json:"tag,omitempty"`
+
+		// Identifier of the test case
+		TcId int `json:"tcId,omitempty"`
+	}
+
+	// Notes a description of the labels used in the test vectors
+	type Notes struct {
+	}
+
+	// AeadTestGroup
+	type AeadTestGroup struct {
+
+		// the IV size in bits
+		IvSize int `json:"ivSize,omitempty"`
+
+		// the keySize in bits
+		KeySize int `json:"keySize,omitempty"`
+
+		// the expected size of the tag in bits
+		TagSize int               `json:"tagSize,omitempty"`
+		Tests   []*AeadTestVector `json:"tests,omitempty"`
+		Type    interface{}       `json:"type,omitempty"`
+	}
+
+	// Root
+	type Root struct {
+
+		// the primitive tested in the test file
+		Algorithm string `json:"algorithm,omitempty"`
+
+		// the version of the test vectors.
+		GeneratorVersion string `json:"generatorVersion,omitempty"`
+
+		// additional documentation
+		Header []string `json:"header,omitempty"`
+
+		// a description of the labels used in the test vectors
+		Notes *Notes `json:"notes,omitempty"`
+
+		// the number of test vectors in this test
+		NumberOfTests int              `json:"numberOfTests,omitempty"`
+		Schema        interface{}      `json:"schema,omitempty"`
+		TestGroups    []*AeadTestGroup `json:"testGroups,omitempty"`
+	}
+
+	testAeadSealOpen := func(t *testing.T, aead cipher.AEAD, tv *AeadTestVector, recoverBadNonce func()) {
+		defer recoverBadNonce()
+
+		// Encrypt the message, then decrypt the new ciphertext and validate
+		// the decrypted message.
+		ciphertext := aead.Seal(nil, decodeHex(tv.Iv), decodeHex(tv.Msg), decodeHex(tv.Aad))
+		msg, err := aead.Open(nil, decodeHex(tv.Iv), ciphertext, decodeHex(tv.Aad))
+		if err != nil {
+			t.Fatalf("#%d: decryption failed: %v", tv.TcId, err)
+		}
+		if got, want := hex.EncodeToString(msg), tv.Msg; got != want {
+			t.Errorf("#%d: bad message after encrypting and decrypting: %s, want %v", tv.TcId, got, want)
+		}
+
+		// Decrypt the provided ciphertext and validate the decrypted message.
+		tv.Ct += tv.Tag // append the tag to the ciphertext
+		msg2, err := aead.Open(nil, decodeHex(tv.Iv), decodeHex(tv.Ct), decodeHex(tv.Aad))
+		wantPass := shouldPass(tv.Result, tv.Flags, nil)
+		if wantPass {
+			if err != nil {
+				t.Errorf("#%d, type: %s, comment: %q, decryption wanted success, got err: %v", tv.TcId, tv.Result, tv.Comment, err)
+			}
+			if got, want := hex.EncodeToString(ciphertext), tv.Ct; got != want {
+				t.Errorf("#%d: ciphertext doesn't match: %s, want=%s", tv.TcId, got, want)
+			}
+			if got, want := hex.EncodeToString(msg2), tv.Msg; got != want {
+				t.Errorf("#%d: bad message after decrypting ciphertext: %s, want %v", tv.TcId, got, want)
+			}
+		} else {
+			if err == nil {
+				t.Errorf("#%d, type: %s, comment: %q, decryption wanted error", tv.TcId, tv.Result, tv.Comment)
+			}
+		}
+	}
+
+	var root Root
+	readTestVector(t, "chacha20_poly1305_test.json", &root)
+	for _, tg := range root.TestGroups {
+		for _, tv := range tg.Tests {
+			aead, err := chacha20poly1305.New(decodeHex(tv.Key))
+			if err != nil {
+				t.Fatalf("#%d: %v", tv.TcId, err)
+			}
+			if tg.TagSize/8 != aead.Overhead() {
+				t.Fatalf("#%d: bad tag length", tv.TcId)
+			}
+			testAeadSealOpen(t, aead, tv, func() {
+				// A bad nonce causes a panic in AEAD.Seal and AEAD.Open,
+				// so should be recovered. Fail the test if it broke for
+				// some other reason.
+				if r := recover(); r != nil {
+					if tg.IvSize/8 == chacha20poly1305.NonceSize {
+						t.Errorf("#%d: unexpected panic", tv.TcId)
+					}
+				}
+			})
+		}
+	}
+}

internal/wycheproof/ecdsa_compat_test.go

@@ -1,10 +1,12 @@
-// Copyright 2019 The Go Authors. All rights reserved.
+// Copyright 2020 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package ecdsa provides an internal version of ecdsa.Verify
-// that is used for the Wycheproof tests.
-package ecdsa
+// +build !go1.15
+
+// ecdsa.VerifyASN1 was added in Go 1.15.
+
+package wycheproof
 
 import (
 	"crypto/ecdsa"
@@ -14,9 +16,7 @@ import (
 	"golang.org/x/crypto/cryptobyte/asn1"
 )
 
-// VerifyASN1 verifies the ASN1 encoded signature, sig, of hash using the
-// public key, pub. Its return value records whether the signature is valid.
-func VerifyASN1(pub *ecdsa.PublicKey, hash, sig []byte) bool {
+func verifyASN1(pub *ecdsa.PublicKey, hash, sig []byte) bool {
 	var (
 		r, s  = &big.Int{}, &big.Int{}
 		inner cryptobyte.String

internal/wycheproof/ecdsa_go115_test.go

@@ -0,0 +1,15 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.15
+
+package wycheproof
+
+import (
+	"crypto/ecdsa"
+)
+
+func verifyASN1(pub *ecdsa.PublicKey, hash, sig []byte) bool {
+	return ecdsa.VerifyASN1(pub, hash, sig)
+}

internal/wycheproof/ecdsa_test.go

@@ -7,8 +7,6 @@ package wycheproof
 import (
 	"crypto/ecdsa"
 	"testing"
-
-	wecdsa "golang.org/x/crypto/internal/wycheproof/internal/ecdsa"
 )
 
 func TestEcdsa(t *testing.T) {
@@ -157,7 +155,7 @@ func TestEcdsa(t *testing.T) {
 			h.Reset()
 			h.Write(decodeHex(sig.Msg))
 			hashed := h.Sum(nil)
-			got := wecdsa.VerifyASN1(pub, hashed, decodeHex(sig.Sig))
+			got := verifyASN1(pub, hashed, decodeHex(sig.Sig))
 			if want := shouldPass(sig.Result, sig.Flags, flagsShouldPass); got != want {
 				t.Errorf("tcid: %d, type: %s, comment: %q, wanted success: %t", sig.TcId, sig.Result, sig.Comment, want)
 			}

internal/wycheproof/hkdf_test.go

@@ -0,0 +1,111 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wycheproof
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"golang.org/x/crypto/hkdf"
+)
+
+func TestHkdf(t *testing.T) {
+
+	// HkdfTestVector
+	type HkdfTestVector struct {
+
+		// A brief description of the test case
+		Comment string `json:"comment,omitempty"`
+
+		// A list of flags
+		Flags []string `json:"flags,omitempty"`
+
+		// the key (input key material)
+		Ikm string `json:"ikm,omitempty"`
+
+		// additional information used in the key derivation
+		Info string `json:"info,omitempty"`
+
+		// the generated bytes (output key material)
+		Okm string `json:"okm,omitempty"`
+
+		// Test result
+		Result string `json:"result,omitempty"`
+
+		// the salt for the key derivation
+		Salt string `json:"salt,omitempty"`
+
+		// the size of the output in bytes
+		Size int `json:"size,omitempty"`
+
+		// Identifier of the test case
+		TcId int `json:"tcId,omitempty"`
+	}
+
+	// Notes a description of the labels used in the test vectors
+	type Notes struct {
+	}
+
+	// HkdfTestGroup
+	type HkdfTestGroup struct {
+
+		// the size of the ikm in bits
+		KeySize int               `json:"keySize,omitempty"`
+		Tests   []*HkdfTestVector `json:"tests,omitempty"`
+		Type    interface{}       `json:"type,omitempty"`
+	}
+
+	// Root
+	type Root struct {
+
+		// the primitive tested in the test file
+		Algorithm string `json:"algorithm,omitempty"`
+
+		// the version of the test vectors.
+		GeneratorVersion string `json:"generatorVersion,omitempty"`
+
+		// additional documentation
+		Header []string `json:"header,omitempty"`
+
+		// a description of the labels used in the test vectors
+		Notes *Notes `json:"notes,omitempty"`
+
+		// the number of test vectors in this test
+		NumberOfTests int              `json:"numberOfTests,omitempty"`
+		Schema        interface{}      `json:"schema,omitempty"`
+		TestGroups    []*HkdfTestGroup `json:"testGroups,omitempty"`
+	}
+
+	fileHashAlgorithms := map[string]string{
+		"hkdf_sha1_test.json":   "SHA-1",
+		"hkdf_sha256_test.json": "SHA-256",
+		"hkdf_sha384_test.json": "SHA-384",
+		"hkdf_sha512_test.json": "SHA-512",
+	}
+
+	for f := range fileHashAlgorithms {
+		var root Root
+		readTestVector(t, f, &root)
+		for _, tg := range root.TestGroups {
+			for _, tv := range tg.Tests {
+				h := parseHash(fileHashAlgorithms[f]).New
+				hkdf := hkdf.New(h, decodeHex(tv.Ikm), decodeHex(tv.Salt), decodeHex(tv.Info))
+				key := make([]byte, tv.Size)
+				wantPass := shouldPass(tv.Result, tv.Flags, nil)
+				_, err := io.ReadFull(hkdf, key)
+				if (err == nil) != wantPass {
+					t.Errorf("tcid: %d, type: %s, comment: %q, wanted success: %t, got: %v", tv.TcId, tv.Result, tv.Comment, wantPass, err)
+				}
+				if err != nil {
+					continue // don't validate output text if reading failed
+				}
+				if got, want := key, decodeHex(tv.Okm); !bytes.Equal(got, want) {
+					t.Errorf("tcid: %d, type: %s, comment: %q, output bytes don't match", tv.TcId, tv.Result, tv.Comment)
+				}
+			}
+		}
+	}
+}

internal/wycheproof/hmac_test.go

@@ -0,0 +1,105 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wycheproof
+
+import (
+	"crypto/hmac"
+	"testing"
+)
+
+func TestHMAC(t *testing.T) {
+	// MacTestVector
+	type MacTestVector struct {
+
+		// A brief description of the test case
+		Comment string `json:"comment,omitempty"`
+
+		// A list of flags
+		Flags []string `json:"flags,omitempty"`
+
+		// the key
+		Key string `json:"key,omitempty"`
+
+		// the plaintext
+		Msg string `json:"msg,omitempty"`
+
+		// Test result
+		Result string `json:"result,omitempty"`
+
+		// the authentication tag
+		Tag string `json:"tag,omitempty"`
+
+		// Identifier of the test case
+		TcId int `json:"tcId,omitempty"`
+	}
+
+	// MacTestGroup
+	type MacTestGroup struct {
+
+		// the keySize in bits
+		KeySize int `json:"keySize,omitempty"`
+
+		// the expected size of the tag in bits
+		TagSize int              `json:"tagSize,omitempty"`
+		Tests   []*MacTestVector `json:"tests,omitempty"`
+		Type    interface{}      `json:"type,omitempty"`
+	}
+
+	// Notes a description of the labels used in the test vectors
+	type Notes struct {
+	}
+
+	// Root
+	type Root struct {
+
+		// the primitive tested in the test file
+		Algorithm string `json:"algorithm,omitempty"`
+
+		// the version of the test vectors.
+		GeneratorVersion string `json:"generatorVersion,omitempty"`
+
+		// additional documentation
+		Header []string `json:"header,omitempty"`
+
+		// a description of the labels used in the test vectors
+		Notes *Notes `json:"notes,omitempty"`
+
+		// the number of test vectors in this test
+		NumberOfTests int             `json:"numberOfTests,omitempty"`
+		Schema        interface{}     `json:"schema,omitempty"`
+		TestGroups    []*MacTestGroup `json:"testGroups,omitempty"`
+	}
+
+	fileHashAlgs := map[string]string{
+		"hmac_sha1_test.json":   "SHA-1",
+		"hmac_sha224_test.json": "SHA-224",
+		"hmac_sha256_test.json": "SHA-256",
+		"hmac_sha384_test.json": "SHA-384",
+		"hmac_sha512_test.json": "SHA-512",
+	}
+
+	for f := range fileHashAlgs {
+		var root Root
+		readTestVector(t, f, &root)
+		for _, tg := range root.TestGroups {
+			h := parseHash(fileHashAlgs[f])
+			// Skip test vectors where the tag length does not equal the
+			// hash length, since crypto/hmac does not support generating
+			// these truncated tags.
+			if tg.TagSize/8 != h.Size() {
+				continue
+			}
+			for _, tv := range tg.Tests {
+				hm := hmac.New(h.New, decodeHex(tv.Key))
+				hm.Write(decodeHex(tv.Msg))
+				tag := hm.Sum(nil)
+				got := hmac.Equal(decodeHex(tv.Tag), tag)
+				if want := shouldPass(tv.Result, tv.Flags, nil); want != got {
+					t.Errorf("%s, tcid: %d, type: %s, comment: %q, unexpected result", f, tv.TcId, tv.Result, tv.Comment)
+				}
+			}
+		}
+	}
+}

internal/wycheproof/wycheproof_test.go

@@ -24,7 +24,7 @@ import (
 	_ "crypto/sha512"
 )
 
-const wycheproofModVer = "v0.0.0-20191126014559-06e5e105eeb9"
+const wycheproofModVer = "v0.0.0-20191219022705-2196000605e4"
 
 var wycheproofTestVectorsDir string
 

openpgp/packet/packet.go

@@ -14,6 +14,7 @@ import (
 	"crypto/rsa"
 	"io"
 	"math/big"
+	"math/bits"
 
 	"golang.org/x/crypto/cast5"
 	"golang.org/x/crypto/openpgp/errors"
@@ -100,33 +101,65 @@ func (r *partialLengthReader) Read(p []byte) (n int, err error) {
 type partialLengthWriter struct {
 	w          io.WriteCloser
 	lengthByte [1]byte
+	sentFirst  bool
+	buf        []byte
 }
 
+// RFC 4880 4.2.2.4: the first partial length MUST be at least 512 octets long.
+const minFirstPartialWrite = 512
+
 func (w *partialLengthWriter) Write(p []byte) (n int, err error) {
-	for len(p) > 0 {
-		for power := uint(14); power < 32; power-- {
-			l := 1 << power
-			if len(p) >= l {
-				w.lengthByte[0] = 224 + uint8(power)
-				_, err = w.w.Write(w.lengthByte[:])
-				if err != nil {
-					return
-				}
-				var m int
-				m, err = w.w.Write(p[:l])
-				n += m
-				if err != nil {
-					return
-				}
-				p = p[l:]
-				break
+	off := 0
+	if !w.sentFirst {
+		if len(w.buf) > 0 || len(p) < minFirstPartialWrite {
+			off = len(w.buf)
+			w.buf = append(w.buf, p...)
+			if len(w.buf) < minFirstPartialWrite {
+				return len(p), nil
 			}
+			p = w.buf
+			w.buf = nil
 		}
+		w.sentFirst = true
 	}
-	return
+
+	power := uint8(30)
+	for len(p) > 0 {
+		l := 1 << power
+		if len(p) < l {
+			power = uint8(bits.Len32(uint32(len(p)))) - 1
+			l = 1 << power
+		}
+		w.lengthByte[0] = 224 + power
+		_, err = w.w.Write(w.lengthByte[:])
+		if err == nil {
+			var m int
+			m, err = w.w.Write(p[:l])
+			n += m
+		}
+		if err != nil {
+			if n < off {
+				return 0, err
+			}
+			return n - off, err
+		}
+		p = p[l:]
+	}
+	return n - off, nil
 }
 
 func (w *partialLengthWriter) Close() error {
+	if len(w.buf) > 0 {
+		// In this case we can't send a 512 byte packet.
+		// Just send what we have.
+		p := w.buf
+		w.sentFirst = true
+		w.buf = nil
+		if _, err := w.Write(p); err != nil {
+			return err
+		}
+	}
+
 	w.lengthByte[0] = 0
 	_, err := w.w.Write(w.lengthByte[:])
 	if err != nil {

openpgp/packet/packet_test.go

@@ -232,7 +232,21 @@ func TestPartialLengths(t *testing.T) {
 			t.Errorf("error from write: %s", err)
 		}
 	}
-	w.Close()
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// The first packet should be at least 512 bytes.
+	first, err := buf.ReadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if plen := 1 << (first & 0x1f); plen < 512 {
+		t.Errorf("first packet too short: got %d want at least %d", plen, 512)
+	}
+	if err := buf.UnreadByte(); err != nil {
+		t.Fatal(err)
+	}
 
 	want := (maxChunkSize * (maxChunkSize + 1)) / 2
 	copyBuf := bytes.NewBuffer(nil)
@@ -253,3 +267,25 @@ func TestPartialLengths(t *testing.T) {
 		}
 	}
 }
+
+func TestPartialLengthsShortWrite(t *testing.T) {
+	buf := bytes.NewBuffer(nil)
+	w := &partialLengthWriter{
+		w: noOpCloser{buf},
+	}
+	data := bytes.Repeat([]byte("a"), 510)
+	if _, err := w.Write(data); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	copyBuf := bytes.NewBuffer(nil)
+	r := &partialLengthReader{buf, 0, true}
+	if _, err := io.Copy(copyBuf, r); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(copyBuf.Bytes(), data) {
+		t.Errorf("got %q want %q", buf.Bytes(), data)
+	}
+}

poly1305/mac_noasm.go

@@ -2,10 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !amd64,!ppc64le gccgo appengine
+// +build !amd64,!ppc64le,!s390x gccgo purego
 
 package poly1305
 
 type mac struct{ macGeneric }
-
-func newMAC(key *[32]byte) mac { return mac{newMACGeneric(key)} }

poly1305/poly1305.go

@@ -26,7 +26,9 @@ const TagSize = 16
 // 16-byte result into out. Authenticating two different messages with the same
 // key allows an attacker to forge messages at will.
 func Sum(out *[16]byte, m []byte, key *[32]byte) {
-	sum(out, m, key)
+	h := New(key)
+	h.Write(m)
+	h.Sum(out[:0])
 }
 
 // Verify returns true if mac is a valid authenticator for m with the given key.
@@ -46,10 +48,9 @@ func Verify(mac *[16]byte, m []byte, key *[32]byte) bool {
 // two different messages with the same key allows an attacker
 // to forge messages at will.
 func New(key *[32]byte) *MAC {
-	return &MAC{
-		mac:       newMAC(key),
-		finalized: false,
-	}
+	m := &MAC{}
+	initialize(key, &m.macState)
+	return m
 }
 
 // MAC is an io.Writer computing an authentication tag
@@ -58,7 +59,7 @@ func New(key *[32]byte) *MAC {
 // MAC cannot be used like common hash.Hash implementations,
 // because using a poly1305 key twice breaks its security.
 // Therefore writing data to a running MAC after calling
-// Sum causes it to panic.
+// Sum or Verify causes it to panic.
 type MAC struct {
 	mac // platform-dependent implementation
 
@@ -71,10 +72,10 @@ func (h *MAC) Size() int { return TagSize }
 // Write adds more data to the running message authentication code.
 // It never returns an error.
 //
-// It must not be called after the first call of Sum.
+// It must not be called after the first call of Sum or Verify.
 func (h *MAC) Write(p []byte) (n int, err error) {
 	if h.finalized {
-		panic("poly1305: write to MAC after Sum")
+		panic("poly1305: write to MAC after Sum or Verify")
 	}
 	return h.mac.Write(p)
 }
@@ -87,3 +88,12 @@ func (h *MAC) Sum(b []byte) []byte {
 	h.finalized = true
 	return append(b, mac[:]...)
 }
+
+// Verify returns whether the authenticator of all data written to
+// the message authentication code matches the expected value.
+func (h *MAC) Verify(expected []byte) bool {
+	var mac [TagSize]byte
+	h.mac.Sum(&mac)
+	h.finalized = true
+	return subtle.ConstantTimeCompare(expected, mac[:]) == 1
+}

poly1305/poly1305_test.go

@@ -6,6 +6,7 @@ package poly1305
 
 import (
 	"crypto/rand"
+	"encoding/binary"
 	"encoding/hex"
 	"flag"
 	"testing"
@@ -15,9 +16,10 @@ import (
 var stressFlag = flag.Bool("stress", false, "run slow stress tests")
 
 type test struct {
-	in  string
-	key string
-	tag string
+	in    string
+	key   string
+	tag   string
+	state string
 }
 
 func (t *test) Input() []byte {
@@ -48,9 +50,33 @@ func (t *test) Tag() [16]byte {
 	return tag
 }
 
+func (t *test) InitialState() [3]uint64 {
+	// state is hex encoded in big-endian byte order
+	if t.state == "" {
+		return [3]uint64{0, 0, 0}
+	}
+	buf, err := hex.DecodeString(t.state)
+	if err != nil {
+		panic(err)
+	}
+	if len(buf) != 3*8 {
+		panic("incorrect state length")
+	}
+	return [3]uint64{
+		binary.BigEndian.Uint64(buf[16:24]),
+		binary.BigEndian.Uint64(buf[8:16]),
+		binary.BigEndian.Uint64(buf[0:8]),
+	}
+}
+
 func testSum(t *testing.T, unaligned bool, sumImpl func(tag *[TagSize]byte, msg []byte, key *[32]byte)) {
 	var tag [16]byte
 	for i, v := range testData {
+		// cannot set initial state before calling sum, so skip those tests
+		if v.InitialState() != [3]uint64{0, 0, 0} {
+			continue
+		}
+
 		in := v.Input()
 		if unaligned {
 			in = unalignBytes(in)
@@ -60,6 +86,30 @@ func testSum(t *testing.T, unaligned bool, sumImpl func(tag *[TagSize]byte, msg
 		if tag != v.Tag() {
 			t.Errorf("%d: expected %x, got %x", i, v.Tag(), tag[:])
 		}
+		if !Verify(&tag, in, &key) {
+			t.Errorf("%d: tag didn't verify", i)
+		}
+		// If the key is zero, the tag will always be zero, independent of the input.
+		if len(in) > 0 && key != [32]byte{} {
+			in[0] ^= 0xff
+			if Verify(&tag, in, &key) {
+				t.Errorf("%d: tag verified after altering the input", i)
+			}
+			in[0] ^= 0xff
+		}
+		// If the input is empty, the tag only depends on the second half of the key.
+		if len(in) > 0 {
+			key[0] ^= 0xff
+			if Verify(&tag, in, &key) {
+				t.Errorf("%d: tag verified after altering the key", i)
+			}
+			key[0] ^= 0xff
+		}
+		tag[0] ^= 0xff
+		if Verify(&tag, in, &key) {
+			t.Errorf("%d: tag verified after altering the tag", i)
+		}
+		tag[0] ^= 0xff
 	}
 }
 
@@ -116,6 +166,9 @@ func testWriteGeneric(t *testing.T, unaligned bool) {
 			input = unalignBytes(input)
 		}
 		h := newMACGeneric(&key)
+		if s := v.InitialState(); s != [3]uint64{0, 0, 0} {
+			h.macState.h = s
+		}
 		n, err := h.Write(input[:len(input)/3])
 		if err != nil || n != len(input[:len(input)/3]) {
 			t.Errorf("#%d: unexpected Write results: n = %d, err = %v", i, n, err)
@@ -141,6 +194,9 @@ func testWrite(t *testing.T, unaligned bool) {
 			input = unalignBytes(input)
 		}
 		h := New(&key)
+		if s := v.InitialState(); s != [3]uint64{0, 0, 0} {
+			h.macState.h = s
+		}
 		n, err := h.Write(input[:len(input)/3])
 		if err != nil || n != len(input[:len(input)/3]) {
 			t.Errorf("#%d: unexpected Write results: n = %d, err = %v", i, n, err)
@@ -150,9 +206,17 @@ func testWrite(t *testing.T, unaligned bool) {
 			t.Errorf("#%d: unexpected Write results: n = %d, err = %v", i, n, err)
 		}
 		h.Sum(out[:0])
-		if tag := v.Tag(); out != tag {
+		tag := v.Tag()
+		if out != tag {
 			t.Errorf("%d: expected %x, got %x", i, tag[:], out[:])
 		}
+		if !h.Verify(tag[:]) {
+			t.Errorf("%d: Verify failed", i)
+		}
+		tag[0] ^= 0xff
+		if h.Verify(tag[:]) {
+			t.Errorf("%d: Verify succeeded after modifying the tag", i)
+		}
 	}
 }
 

poly1305/sum_amd64.go

@@ -2,24 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build amd64,!gccgo,!appengine
+// +build !gccgo,!purego
 
 package poly1305
 
 //go:noescape
 func update(state *macState, msg []byte)
 
-func sum(out *[16]byte, m []byte, key *[32]byte) {
-	h := newMAC(key)
-	h.Write(m)
-	h.Sum(out)
-}
-
-func newMAC(key *[32]byte) (h mac) {
-	initialize(key, &h.r, &h.s)
-	return
-}
-
 // mac is a wrapper for macGeneric that redirects calls that would have gone to
 // updateGeneric to update.
 //

poly1305/sum_amd64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build amd64,!gccgo,!appengine
+// +build !gccgo,!purego
 
 #include "textflag.h"
 

poly1305/sum_generic.go

@@ -31,16 +31,18 @@ func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) {
 	h.Sum(out)
 }
 
-func newMACGeneric(key *[32]byte) (h macGeneric) {
-	initialize(key, &h.r, &h.s)
-	return
+func newMACGeneric(key *[32]byte) macGeneric {
+	m := macGeneric{}
+	initialize(key, &m.macState)
+	return m
 }
 
 // macState holds numbers in saturated 64-bit little-endian limbs. That is,
 // the value of [x0, x1, x2] is x[0] + x[1] * 2⁶⁴ + x[2] * 2¹²⁸.
 type macState struct {
 	// h is the main accumulator. It is to be interpreted modulo 2¹³⁰ - 5, but
-	// can grow larger during and after rounds.
+	// can grow larger during and after rounds. It must, however, remain below
+	// 2 * (2¹³⁰ - 5).
 	h [3]uint64
 	// r and s are the private key components.
 	r [2]uint64
@@ -97,11 +99,12 @@ const (
 	rMask1 = 0x0FFFFFFC0FFFFFFC
 )
 
-func initialize(key *[32]byte, r, s *[2]uint64) {
-	r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0
-	r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1
-	s[0] = binary.LittleEndian.Uint64(key[16:24])
-	s[1] = binary.LittleEndian.Uint64(key[24:32])
+// initialize loads the 256-bit key into the two 128-bit secret values r and s.
+func initialize(key *[32]byte, m *macState) {
+	m.r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0
+	m.r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1
+	m.s[0] = binary.LittleEndian.Uint64(key[16:24])
+	m.s[1] = binary.LittleEndian.Uint64(key[24:32])
 }
 
 // uint128 holds a 128-bit number as two 64-bit limbs, for use with the

poly1305/sum_noasm.go

@@ -1,13 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!go1.11 !amd64,!s390x,!ppc64le gccgo appengine nacl
-
-package poly1305
-
-func sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
-	h := newMAC(key)
-	h.Write(msg)
-	h.Sum(out)
-}

poly1305/sum_ppc64le.go

@@ -2,24 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build ppc64le,!gccgo,!appengine
+// +build !gccgo,!purego
 
 package poly1305
 
 //go:noescape
 func update(state *macState, msg []byte)
 
-func sum(out *[16]byte, m []byte, key *[32]byte) {
-	h := newMAC(key)
-	h.Write(m)
-	h.Sum(out)
-}
-
-func newMAC(key *[32]byte) (h mac) {
-	initialize(key, &h.r, &h.s)
-	return
-}
-
 // mac is a wrapper for macGeneric that redirects calls that would have gone to
 // updateGeneric to update.
 //

poly1305/sum_ppc64le.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build ppc64le,!gccgo,!appengine
+// +build !gccgo,!purego
 
 #include "textflag.h"
 

poly1305/sum_s390x.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build s390x,go1.11,!gccgo,!appengine
+// +build !gccgo,!purego
 
 package poly1305
 
@@ -10,30 +10,66 @@ import (
 	"golang.org/x/sys/cpu"
 )
 
-// poly1305vx is an assembly implementation of Poly1305 that uses vector
+// updateVX is an assembly implementation of Poly1305 that uses vector
 // instructions. It must only be called if the vector facility (vx) is
 // available.
 //go:noescape
-func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+func updateVX(state *macState, msg []byte)
 
-// poly1305vmsl is an assembly implementation of Poly1305 that uses vector
-// instructions, including VMSL. It must only be called if the vector facility (vx) is
-// available and if VMSL is supported.
-//go:noescape
-func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+// mac is a replacement for macGeneric that uses a larger buffer and redirects
+// calls that would have gone to updateGeneric to updateVX if the vector
+// facility is installed.
+//
+// A larger buffer is required for good performance because the vector
+// implementation has a higher fixed cost per call than the generic
+// implementation.
+type mac struct {
+	macState
 
-func sum(out *[16]byte, m []byte, key *[32]byte) {
-	if cpu.S390X.HasVX {
-		var mPtr *byte
-		if len(m) > 0 {
-			mPtr = &m[0]
-		}
-		if cpu.S390X.HasVXE && len(m) > 256 {
-			poly1305vmsl(out, mPtr, uint64(len(m)), key)
-		} else {
-			poly1305vx(out, mPtr, uint64(len(m)), key)
-		}
-	} else {
-		sumGeneric(out, m, key)
-	}
+	buffer [16 * TagSize]byte // size must be a multiple of block size (16)
+	offset int
+}
+
+func (h *mac) Write(p []byte) (int, error) {
+	nn := len(p)
+	if h.offset > 0 {
+		n := copy(h.buffer[h.offset:], p)
+		if h.offset+n < len(h.buffer) {
+			h.offset += n
+			return nn, nil
+		}
+		p = p[n:]
+		h.offset = 0
+		if cpu.S390X.HasVX {
+			updateVX(&h.macState, h.buffer[:])
+		} else {
+			updateGeneric(&h.macState, h.buffer[:])
+		}
+	}
+
+	tail := len(p) % len(h.buffer) // number of bytes to copy into buffer
+	body := len(p) - tail          // number of bytes to process now
+	if body > 0 {
+		if cpu.S390X.HasVX {
+			updateVX(&h.macState, p[:body])
+		} else {
+			updateGeneric(&h.macState, p[:body])
+		}
+	}
+	h.offset = copy(h.buffer[:], p[body:]) // copy tail bytes - can be 0
+	return nn, nil
+}
+
+func (h *mac) Sum(out *[TagSize]byte) {
+	state := h.macState
+	remainder := h.buffer[:h.offset]
+
+	// Use the generic implementation if we have 2 or fewer blocks left
+	// to sum. The vector implementation has a higher startup time.
+	if cpu.S390X.HasVX && len(remainder) > 2*TagSize {
+		updateVX(&state, remainder)
+	} else if len(remainder) > 0 {
+		updateGeneric(&state, remainder)
+	}
+	finalize(out, &state.h, &state.s)
 }

poly1305/sum_s390x.s

@@ -2,115 +2,187 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build s390x,go1.11,!gccgo,!appengine
+// +build !gccgo,!purego
 
 #include "textflag.h"
 
-// Implementation of Poly1305 using the vector facility (vx).
+// This implementation of Poly1305 uses the vector facility (vx)
+// to process up to 2 blocks (32 bytes) per iteration using an
+// algorithm based on the one described in:
+//
+// NEON crypto, Daniel J. Bernstein & Peter Schwabe
+// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
+//
+// This algorithm uses 5 26-bit limbs to represent a 130-bit
+// value. These limbs are, for the most part, zero extended and
+// placed into 64-bit vector register elements. Each vector
+// register is 128-bits wide and so holds 2 of these elements.
+// Using 26-bit limbs allows us plenty of headroom to accomodate
+// accumulations before and after multiplication without
+// overflowing either 32-bits (before multiplication) or 64-bits
+// (after multiplication).
+//
+// In order to parallelise the operations required to calculate
+// the sum we use two separate accumulators and then sum those
+// in an extra final step. For compatibility with the generic
+// implementation we perform this summation at the end of every
+// updateVX call.
+//
+// To use two accumulators we must multiply the message blocks
+// by r² rather than r. Only the final message block should be
+// multiplied by r.
+//
+// Example:
+//
+// We want to calculate the sum (h) for a 64 byte message (m):
+//
+//   h = m[0:16]r⁴ + m[16:32]r³ + m[32:48]r² + m[48:64]r
+//
+// To do this we split the calculation into the even indices
+// and odd indices of the message. These form our SIMD 'lanes':
+//
+//   h = m[ 0:16]r⁴ + m[32:48]r² +   <- lane 0
+//       m[16:32]r³ + m[48:64]r      <- lane 1
+//
+// To calculate this iteratively we refactor so that both lanes
+// are written in terms of r² and r:
+//
+//   h = (m[ 0:16]r² + m[32:48])r² + <- lane 0
+//       (m[16:32]r² + m[48:64])r    <- lane 1
+//                ^             ^
+//                |             coefficients for second iteration
+//                coefficients for first iteration
+//
+// So in this case we would have two iterations. In the first
+// both lanes are multiplied by r². In the second only the
+// first lane is multiplied by r² and the second lane is
+// instead multiplied by r. This gives use the odd and even
+// powers of r that we need from the original equation.
+//
+// Notation:
+//
+//   h - accumulator
+//   r - key
+//   m - message
+//
+//   [a, b]       - SIMD register holding two 64-bit values
+//   [a, b, c, d] - SIMD register holding four 32-bit values
+//   xᵢ[n]        - limb n of variable x with bit width i
+//
+// Limbs are expressed in little endian order, so for 26-bit
+// limbs x₂₆[4] will be the most significant limb and x₂₆[0]
+// will be the least significant limb.
 
-// constants
-#define MOD26 V0
-#define EX0   V1
-#define EX1   V2
-#define EX2   V3
+// masking constants
+#define MOD24 V0 // [0x0000000000ffffff, 0x0000000000ffffff] - mask low 24-bits
+#define MOD26 V1 // [0x0000000003ffffff, 0x0000000003ffffff] - mask low 26-bits
 
-// temporaries
-#define T_0 V4
-#define T_1 V5
-#define T_2 V6
-#define T_3 V7
-#define T_4 V8
+// expansion constants (see EXPAND macro)
+#define EX0 V2
+#define EX1 V3
+#define EX2 V4
 
-// key (r)
-#define R_0  V9
-#define R_1  V10
-#define R_2  V11
-#define R_3  V12
-#define R_4  V13
-#define R5_1 V14
-#define R5_2 V15
-#define R5_3 V16
-#define R5_4 V17
-#define RSAVE_0 R5
-#define RSAVE_1 R6
-#define RSAVE_2 R7
-#define RSAVE_3 R8
-#define RSAVE_4 R9
-#define R5SAVE_1 V28
-#define R5SAVE_2 V29
-#define R5SAVE_3 V30
-#define R5SAVE_4 V31
+// key (r², r or 1 depending on context)
+#define R_0 V5
+#define R_1 V6
+#define R_2 V7
+#define R_3 V8
+#define R_4 V9
 
-// message block
-#define F_0 V18
-#define F_1 V19
-#define F_2 V20
-#define F_3 V21
-#define F_4 V22
+// precalculated coefficients (5r², 5r or 0 depending on context)
+#define R5_1 V10
+#define R5_2 V11
+#define R5_3 V12
+#define R5_4 V13
 
-// accumulator
-#define H_0 V23
-#define H_1 V24
-#define H_2 V25
-#define H_3 V26
-#define H_4 V27
+// message block (m)
+#define M_0 V14
+#define M_1 V15
+#define M_2 V16
+#define M_3 V17
+#define M_4 V18
 
-GLOBL ·keyMask<>(SB), RODATA, $16
-DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f
-DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f
+// accumulator (h)
+#define H_0 V19
+#define H_1 V20
+#define H_2 V21
+#define H_3 V22
+#define H_4 V23
 
-GLOBL ·bswapMask<>(SB), RODATA, $16
-DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908
-DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100
+// temporary registers (for short-lived values)
+#define T_0 V24
+#define T_1 V25
+#define T_2 V26
+#define T_3 V27
+#define T_4 V28
 
-GLOBL ·constants<>(SB), RODATA, $64
-// MOD26
-DATA ·constants<>+0(SB)/8, $0x3ffffff
-DATA ·constants<>+8(SB)/8, $0x3ffffff
+GLOBL ·constants<>(SB), RODATA, $0x30
 // EX0
-DATA ·constants<>+16(SB)/8, $0x0006050403020100
-DATA ·constants<>+24(SB)/8, $0x1016151413121110
+DATA ·constants<>+0x00(SB)/8, $0x0006050403020100
+DATA ·constants<>+0x08(SB)/8, $0x1016151413121110
 // EX1
-DATA ·constants<>+32(SB)/8, $0x060c0b0a09080706
-DATA ·constants<>+40(SB)/8, $0x161c1b1a19181716
+DATA ·constants<>+0x10(SB)/8, $0x060c0b0a09080706
+DATA ·constants<>+0x18(SB)/8, $0x161c1b1a19181716
 // EX2
-DATA ·constants<>+48(SB)/8, $0x0d0d0d0d0d0f0e0d
-DATA ·constants<>+56(SB)/8, $0x1d1d1d1d1d1f1e1d
+DATA ·constants<>+0x20(SB)/8, $0x0d0d0d0d0d0f0e0d
+DATA ·constants<>+0x28(SB)/8, $0x1d1d1d1d1d1f1e1d
 
-// h = (f*g) % (2**130-5) [partial reduction]
+// MULTIPLY multiplies each lane of f and g, partially reduced
+// modulo 2¹³⁰ - 5. The result, h, consists of partial products
+// in each lane that need to be reduced further to produce the
+// final result.
+//
+//   h₁₃₀ = (f₁₃₀g₁₃₀) % 2¹³⁰ + (5f₁₃₀g₁₃₀) / 2¹³⁰
+//
+// Note that the multiplication by 5 of the high bits is
+// achieved by precalculating the multiplication of four of the
+// g coefficients by 5. These are g51-g54.
 #define MULTIPLY(f0, f1, f2, f3, f4, g0, g1, g2, g3, g4, g51, g52, g53, g54, h0, h1, h2, h3, h4) \
 	VMLOF  f0, g0, h0        \
-	VMLOF  f0, g1, h1        \
-	VMLOF  f0, g2, h2        \
 	VMLOF  f0, g3, h3        \
+	VMLOF  f0, g1, h1        \
 	VMLOF  f0, g4, h4        \
+	VMLOF  f0, g2, h2        \
 	VMLOF  f1, g54, T_0      \
-	VMLOF  f1, g0, T_1       \
-	VMLOF  f1, g1, T_2       \
 	VMLOF  f1, g2, T_3       \
+	VMLOF  f1, g0, T_1       \
 	VMLOF  f1, g3, T_4       \
+	VMLOF  f1, g1, T_2       \
 	VMALOF f2, g53, h0, h0   \
-	VMALOF f2, g54, h1, h1   \
-	VMALOF f2, g0, h2, h2    \
 	VMALOF f2, g1, h3, h3    \
+	VMALOF f2, g54, h1, h1   \
 	VMALOF f2, g2, h4, h4    \
+	VMALOF f2, g0, h2, h2    \
 	VMALOF f3, g52, T_0, T_0 \
-	VMALOF f3, g53, T_1, T_1 \
-	VMALOF f3, g54, T_2, T_2 \
 	VMALOF f3, g0, T_3, T_3  \
+	VMALOF f3, g53, T_1, T_1 \
 	VMALOF f3, g1, T_4, T_4  \
+	VMALOF f3, g54, T_2, T_2 \
 	VMALOF f4, g51, h0, h0   \
-	VMALOF f4, g52, h1, h1   \
-	VMALOF f4, g53, h2, h2   \
 	VMALOF f4, g54, h3, h3   \
+	VMALOF f4, g52, h1, h1   \
 	VMALOF f4, g0, h4, h4    \
+	VMALOF f4, g53, h2, h2   \
 	VAG    T_0, h0, h0       \
-	VAG    T_1, h1, h1       \
-	VAG    T_2, h2, h2       \
 	VAG    T_3, h3, h3       \
-	VAG    T_4, h4, h4
+	VAG    T_1, h1, h1       \
+	VAG    T_4, h4, h4       \
+	VAG    T_2, h2, h2
 
-// carry h0->h1 h3->h4, h1->h2 h4->h0, h0->h1 h2->h3, h3->h4
+// REDUCE performs the following carry operations in four
+// stages, as specified in Bernstein & Schwabe:
+//
+//   1: h₂₆[0]->h₂₆[1] h₂₆[3]->h₂₆[4]
+//   2: h₂₆[1]->h₂₆[2] h₂₆[4]->h₂₆[0]
+//   3: h₂₆[0]->h₂₆[1] h₂₆[2]->h₂₆[3]
+//   4: h₂₆[3]->h₂₆[4]
+//
+// The result is that all of the limbs are limited to 26-bits
+// except for h₂₆[1] and h₂₆[4] which are limited to 27-bits.
+//
+// Note that although each limb is aligned at 26-bit intervals
+// they may contain values that exceed 2²⁶ - 1, hence the need
+// to carry the excess bits in each limb.
 #define REDUCE(h0, h1, h2, h3, h4) \
 	VESRLG $26, h0, T_0  \
 	VESRLG $26, h3, T_1  \
@@ -136,144 +208,155 @@ DATA ·constants<>+56(SB)/8, $0x1d1d1d1d1d1f1e1d
 	VN     MOD26, h3, h3 \
 	VAG    T_2, h4, h4
 
-// expand in0 into d[0] and in1 into d[1]
+// EXPAND splits the 128-bit little-endian values in0 and in1
+// into 26-bit big-endian limbs and places the results into
+// the first and second lane of d₂₆[0:4] respectively.
+//
+// The EX0, EX1 and EX2 constants are arrays of byte indices
+// for permutation. The permutation both reverses the bytes
+// in the input and ensures the bytes are copied into the
+// destination limb ready to be shifted into their final
+// position.
 #define EXPAND(in0, in1, d0, d1, d2, d3, d4) \
-	VGBM   $0x0707, d1       \ // d1=tmp
-	VPERM  in0, in1, EX2, d4 \
 	VPERM  in0, in1, EX0, d0 \
 	VPERM  in0, in1, EX1, d2 \
-	VN     d1, d4, d4        \
+	VPERM  in0, in1, EX2, d4 \
 	VESRLG $26, d0, d1       \
 	VESRLG $30, d2, d3       \
 	VESRLG $4, d2, d2        \
-	VN     MOD26, d0, d0     \
-	VN     MOD26, d1, d1     \
-	VN     MOD26, d2, d2     \
-	VN     MOD26, d3, d3
+	VN     MOD26, d0, d0     \ // [in0₂₆[0], in1₂₆[0]]
+	VN     MOD26, d3, d3     \ // [in0₂₆[3], in1₂₆[3]]
+	VN     MOD26, d1, d1     \ // [in0₂₆[1], in1₂₆[1]]
+	VN     MOD24, d4, d4     \ // [in0₂₆[4], in1₂₆[4]]
+	VN     MOD26, d2, d2     // [in0₂₆[2], in1₂₆[2]]
 
-// pack h4:h0 into h1:h0 (no carry)
-#define PACK(h0, h1, h2, h3, h4) \
-	VESLG $26, h1, h1  \
-	VESLG $26, h3, h3  \
-	VO    h0, h1, h0   \
-	VO    h2, h3, h2   \
-	VESLG $4, h2, h2   \
-	VLEIB $7, $48, h1  \
-	VSLB  h1, h2, h2   \
-	VO    h0, h2, h0   \
-	VLEIB $7, $104, h1 \
-	VSLB  h1, h4, h3   \
-	VO    h3, h0, h0   \
-	VLEIB $7, $24, h1  \
-	VSRLB h1, h4, h1
+// func updateVX(state *macState, msg []byte)
+TEXT ·updateVX(SB), NOSPLIT, $0
+	MOVD state+0(FP), R1
+	LMG  msg+8(FP), R2, R3 // R2=msg_base, R3=msg_len
 
-// if h > 2**130-5 then h -= 2**130-5
-#define MOD(h0, h1, t0, t1, t2) \
-	VZERO t0          \
-	VLEIG $1, $5, t0  \
-	VACCQ h0, t0, t1  \
-	VAQ   h0, t0, t0  \
-	VONE  t2          \
-	VLEIG $1, $-4, t2 \
-	VAQ   t2, t1, t1  \
-	VACCQ h1, t1, t1  \
-	VONE  t2          \
-	VAQ   t2, t1, t1  \
-	VN    h0, t1, t2  \
-	VNC   t0, t1, t1  \
-	VO    t1, t2, h0
-
-// func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]key)
-TEXT ·poly1305vx(SB), $0-32
-	// This code processes up to 2 blocks (32 bytes) per iteration
-	// using the algorithm described in:
-	// NEON crypto, Daniel J. Bernstein & Peter Schwabe
-	// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
-	LMG out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key
-
-	// load MOD26, EX0, EX1 and EX2
+	// load EX0, EX1 and EX2
 	MOVD $·constants<>(SB), R5
-	VLM  (R5), MOD26, EX2
+	VLM  (R5), EX0, EX2
 
-	// setup r
-	VL   (R4), T_0
-	MOVD $·keyMask<>(SB), R6
-	VL   (R6), T_1
-	VN   T_0, T_1, T_0
-	EXPAND(T_0, T_0, R_0, R_1, R_2, R_3, R_4)
+	// generate masks
+	VGMG $(64-24), $63, MOD24 // [0x00ffffff, 0x00ffffff]
+	VGMG $(64-26), $63, MOD26 // [0x03ffffff, 0x03ffffff]
 
-	// setup r*5
-	VLEIG $0, $5, T_0
-	VLEIG $1, $5, T_0
+	// load h (accumulator) and r (key) from state
+	VZERO T_1               // [0, 0]
+	VL    0(R1), T_0        // [h₆₄[0], h₆₄[1]]
+	VLEG  $0, 16(R1), T_1   // [h₆₄[2], 0]
+	VL    24(R1), T_2       // [r₆₄[0], r₆₄[1]]
+	VPDI  $0, T_0, T_2, T_3 // [h₆₄[0], r₆₄[0]]
+	VPDI  $5, T_0, T_2, T_4 // [h₆₄[1], r₆₄[1]]
 
-	// store r (for final block)
-	VMLOF T_0, R_1, R5SAVE_1
-	VMLOF T_0, R_2, R5SAVE_2
-	VMLOF T_0, R_3, R5SAVE_3
-	VMLOF T_0, R_4, R5SAVE_4
-	VLGVG $0, R_0, RSAVE_0
-	VLGVG $0, R_1, RSAVE_1
-	VLGVG $0, R_2, RSAVE_2
-	VLGVG $0, R_3, RSAVE_3
-	VLGVG $0, R_4, RSAVE_4
+	// unpack h and r into 26-bit limbs
+	// note: h₆₄[2] may have the low 3 bits set, so h₂₆[4] is a 27-bit value
+	VN     MOD26, T_3, H_0            // [h₂₆[0], r₂₆[0]]
+	VZERO  H_1                        // [0, 0]
+	VZERO  H_3                        // [0, 0]
+	VGMG   $(64-12-14), $(63-12), T_0 // [0x03fff000, 0x03fff000] - 26-bit mask with low 12 bits masked out
+	VESLG  $24, T_1, T_1              // [h₆₄[2]<<24, 0]
+	VERIMG $-26&63, T_3, MOD26, H_1   // [h₂₆[1], r₂₆[1]]
+	VESRLG $+52&63, T_3, H_2          // [h₂₆[2], r₂₆[2]] - low 12 bits only
+	VERIMG $-14&63, T_4, MOD26, H_3   // [h₂₆[1], r₂₆[1]]
+	VESRLG $40, T_4, H_4              // [h₂₆[4], r₂₆[4]] - low 24 bits only
+	VERIMG $+12&63, T_4, T_0, H_2     // [h₂₆[2], r₂₆[2]] - complete
+	VO     T_1, H_4, H_4              // [h₂₆[4], r₂₆[4]] - complete
 
-	// skip r**2 calculation
+	// replicate r across all 4 vector elements
+	VREPF $3, H_0, R_0 // [r₂₆[0], r₂₆[0], r₂₆[0], r₂₆[0]]
+	VREPF $3, H_1, R_1 // [r₂₆[1], r₂₆[1], r₂₆[1], r₂₆[1]]
+	VREPF $3, H_2, R_2 // [r₂₆[2], r₂₆[2], r₂₆[2], r₂₆[2]]
+	VREPF $3, H_3, R_3 // [r₂₆[3], r₂₆[3], r₂₆[3], r₂₆[3]]
+	VREPF $3, H_4, R_4 // [r₂₆[4], r₂₆[4], r₂₆[4], r₂₆[4]]
+
+	// zero out lane 1 of h
+	VLEIG $1, $0, H_0 // [h₂₆[0], 0]
+	VLEIG $1, $0, H_1 // [h₂₆[1], 0]
+	VLEIG $1, $0, H_2 // [h₂₆[2], 0]
+	VLEIG $1, $0, H_3 // [h₂₆[3], 0]
+	VLEIG $1, $0, H_4 // [h₂₆[4], 0]
+
+	// calculate 5r (ignore least significant limb)
+	VREPIF $5, T_0
+	VMLF   T_0, R_1, R5_1 // [5r₂₆[1], 5r₂₆[1], 5r₂₆[1], 5r₂₆[1]]
+	VMLF   T_0, R_2, R5_2 // [5r₂₆[2], 5r₂₆[2], 5r₂₆[2], 5r₂₆[2]]
+	VMLF   T_0, R_3, R5_3 // [5r₂₆[3], 5r₂₆[3], 5r₂₆[3], 5r₂₆[3]]
+	VMLF   T_0, R_4, R5_4 // [5r₂₆[4], 5r₂₆[4], 5r₂₆[4], 5r₂₆[4]]
+
+	// skip r² calculation if we are only calculating one block
 	CMPBLE R3, $16, skip
 
-	// calculate r**2
-	MULTIPLY(R_0, R_1, R_2, R_3, R_4, R_0, R_1, R_2, R_3, R_4, R5SAVE_1, R5SAVE_2, R5SAVE_3, R5SAVE_4, H_0, H_1, H_2, H_3, H_4)
-	REDUCE(H_0, H_1, H_2, H_3, H_4)
-	VLEIG $0, $5, T_0
-	VLEIG $1, $5, T_0
-	VMLOF T_0, H_1, R5_1
-	VMLOF T_0, H_2, R5_2
-	VMLOF T_0, H_3, R5_3
-	VMLOF T_0, H_4, R5_4
-	VLR   H_0, R_0
-	VLR   H_1, R_1
-	VLR   H_2, R_2
-	VLR   H_3, R_3
-	VLR   H_4, R_4
+	// calculate r²
+	MULTIPLY(R_0, R_1, R_2, R_3, R_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, M_0, M_1, M_2, M_3, M_4)
+	REDUCE(M_0, M_1, M_2, M_3, M_4)
+	VGBM   $0x0f0f, T_0
+	VERIMG $0, M_0, T_0, R_0 // [r₂₆[0], r²₂₆[0], r₂₆[0], r²₂₆[0]]
+	VERIMG $0, M_1, T_0, R_1 // [r₂₆[1], r²₂₆[1], r₂₆[1], r²₂₆[1]]
+	VERIMG $0, M_2, T_0, R_2 // [r₂₆[2], r²₂₆[2], r₂₆[2], r²₂₆[2]]
+	VERIMG $0, M_3, T_0, R_3 // [r₂₆[3], r²₂₆[3], r₂₆[3], r²₂₆[3]]
+	VERIMG $0, M_4, T_0, R_4 // [r₂₆[4], r²₂₆[4], r₂₆[4], r²₂₆[4]]
 
-	// initialize h
-	VZERO H_0
-	VZERO H_1
-	VZERO H_2
-	VZERO H_3
-	VZERO H_4
+	// calculate 5r² (ignore least significant limb)
+	VREPIF $5, T_0
+	VMLF   T_0, R_1, R5_1 // [5r₂₆[1], 5r²₂₆[1], 5r₂₆[1], 5r²₂₆[1]]
+	VMLF   T_0, R_2, R5_2 // [5r₂₆[2], 5r²₂₆[2], 5r₂₆[2], 5r²₂₆[2]]
+	VMLF   T_0, R_3, R5_3 // [5r₂₆[3], 5r²₂₆[3], 5r₂₆[3], 5r²₂₆[3]]
+	VMLF   T_0, R_4, R5_4 // [5r₂₆[4], 5r²₂₆[4], 5r₂₆[4], 5r²₂₆[4]]
 
 loop:
-	CMPBLE R3, $32, b2
-	VLM    (R2), T_0, T_1
-	SUB    $32, R3
-	MOVD   $32(R2), R2
-	EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4)
-	VLEIB  $4, $1, F_4
-	VLEIB  $12, $1, F_4
+	CMPBLE R3, $32, b2 // 2 or fewer blocks remaining, need to change key coefficients
+
+	// load next 2 blocks from message
+	VLM (R2), T_0, T_1
+
+	// update message slice
+	SUB  $32, R3
+	MOVD $32(R2), R2
+
+	// unpack message blocks into 26-bit big-endian limbs
+	EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4)
+
+	// add 2¹²⁸ to each message block value
+	VLEIB $4, $1, M_4
+	VLEIB $12, $1, M_4
 
 multiply:
-	VAG    H_0, F_0, F_0
-	VAG    H_1, F_1, F_1
-	VAG    H_2, F_2, F_2
-	VAG    H_3, F_3, F_3
-	VAG    H_4, F_4, F_4
-	MULTIPLY(F_0, F_1, F_2, F_3, F_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, H_0, H_1, H_2, H_3, H_4)
+	// accumulate the incoming message
+	VAG H_0, M_0, M_0
+	VAG H_3, M_3, M_3
+	VAG H_1, M_1, M_1
+	VAG H_4, M_4, M_4
+	VAG H_2, M_2, M_2
+
+	// multiply the accumulator by the key coefficient
+	MULTIPLY(M_0, M_1, M_2, M_3, M_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, H_0, H_1, H_2, H_3, H_4)
+
+	// carry and partially reduce the partial products
 	REDUCE(H_0, H_1, H_2, H_3, H_4)
+
 	CMPBNE R3, $0, loop
 
 finish:
-	// sum vectors
+	// sum lane 0 and lane 1 and put the result in lane 1
 	VZERO  T_0
 	VSUMQG H_0, T_0, H_0
-	VSUMQG H_1, T_0, H_1
-	VSUMQG H_2, T_0, H_2
 	VSUMQG H_3, T_0, H_3
+	VSUMQG H_1, T_0, H_1
 	VSUMQG H_4, T_0, H_4
+	VSUMQG H_2, T_0, H_2
 
-	// h may be >= 2*(2**130-5) so we need to reduce it again
+	// reduce again after summation
+	// TODO(mundaym): there might be a more efficient way to do this
+	// now that we only have 1 active lane. For example, we could
+	// simultaneously pack the values as we reduce them.
 	REDUCE(H_0, H_1, H_2, H_3, H_4)
 
-	// carry h1->h4
+	// carry h[1] through to h[4] so that only h[4] can exceed 2²⁶ - 1
+	// TODO(mundaym): in testing this final carry was unnecessary.
+	// Needs a proof before it can be removed though.
 	VESRLG $26, H_1, T_1
 	VN     MOD26, H_1, H_1
 	VAQ    T_1, H_2, H_2
@@ -284,95 +367,137 @@ finish:
 	VN     MOD26, H_3, H_3
 	VAQ    T_3, H_4, H_4
 
-	// h is now < 2*(2**130-5)
-	// pack h into h1 (hi) and h0 (lo)
-	PACK(H_0, H_1, H_2, H_3, H_4)
-
-	// if h > 2**130-5 then h -= 2**130-5
-	MOD(H_0, H_1, T_0, T_1, T_2)
-
-	// h += s
-	MOVD  $·bswapMask<>(SB), R5
-	VL    (R5), T_1
-	VL    16(R4), T_0
-	VPERM T_0, T_0, T_1, T_0    // reverse bytes (to big)
-	VAQ   T_0, H_0, H_0
-	VPERM H_0, H_0, T_1, H_0    // reverse bytes (to little)
-	VST   H_0, (R1)
+	// h is now < 2(2¹³⁰ - 5)
+	// Pack each lane in h₂₆[0:4] into h₁₂₈[0:1].
+	VESLG $26, H_1, H_1
+	VESLG $26, H_3, H_3
+	VO    H_0, H_1, H_0
+	VO    H_2, H_3, H_2
+	VESLG $4, H_2, H_2
+	VLEIB $7, $48, H_1
+	VSLB  H_1, H_2, H_2
+	VO    H_0, H_2, H_0
+	VLEIB $7, $104, H_1
+	VSLB  H_1, H_4, H_3
+	VO    H_3, H_0, H_0
+	VLEIB $7, $24, H_1
+	VSRLB H_1, H_4, H_1
 
+	// update state
+	VSTEG $1, H_0, 0(R1)
+	VSTEG $0, H_0, 8(R1)
+	VSTEG $1, H_1, 16(R1)
 	RET
 
-b2:
+b2:  // 2 or fewer blocks remaining
 	CMPBLE R3, $16, b1
 
-	// 2 blocks remaining
-	SUB    $17, R3
-	VL     (R2), T_0
-	VLL    R3, 16(R2), T_1
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, T_1
-	EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4)
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $12, $1, F_4
-	VLEIB  $4, $1, F_4
+	// Load the 2 remaining blocks (17-32 bytes remaining).
+	MOVD $-17(R3), R0    // index of final byte to load modulo 16
+	VL   (R2), T_0       // load full 16 byte block
+	VLL  R0, 16(R2), T_1 // load final (possibly partial) block and pad with zeros to 16 bytes
 
-	// setup [r²,r]
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, RSAVE_3, R_3
-	VLVGG $1, RSAVE_4, R_4
-	VPDI  $0, R5_1, R5SAVE_1, R5_1
-	VPDI  $0, R5_2, R5SAVE_2, R5_2
-	VPDI  $0, R5_3, R5SAVE_3, R5_3
-	VPDI  $0, R5_4, R5SAVE_4, R5_4
+	// The Poly1305 algorithm requires that a 1 bit be appended to
+	// each message block. If the final block is less than 16 bytes
+	// long then it is easiest to insert the 1 before the message
+	// block is split into 26-bit limbs. If, on the other hand, the
+	// final message block is 16 bytes long then we append the 1 bit
+	// after expansion as normal.
+	MOVBZ  $1, R0
+	MOVD   $-16(R3), R3   // index of byte in last block to insert 1 at (could be 16)
+	CMPBEQ R3, $16, 2(PC) // skip the insertion if the final block is 16 bytes long
+	VLVGB  R3, R0, T_1    // insert 1 into the byte at index R3
+
+	// Split both blocks into 26-bit limbs in the appropriate lanes.
+	EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4)
+
+	// Append a 1 byte to the end of the second to last block.
+	VLEIB $4, $1, M_4
+
+	// Append a 1 byte to the end of the last block only if it is a
+	// full 16 byte block.
+	CMPBNE R3, $16, 2(PC)
+	VLEIB  $12, $1, M_4
+
+	// Finally, set up the coefficients for the final multiplication.
+	// We have previously saved r and 5r in the 32-bit even indexes
+	// of the R_[0-4] and R5_[1-4] coefficient registers.
+	//
+	// We want lane 0 to be multiplied by r² so that can be kept the
+	// same. We want lane 1 to be multiplied by r so we need to move
+	// the saved r value into the 32-bit odd index in lane 1 by
+	// rotating the 64-bit lane by 32.
+	VGBM   $0x00ff, T_0         // [0, 0xffffffffffffffff] - mask lane 1 only
+	VERIMG $32, R_0, T_0, R_0   // [_,  r²₂₆[0], _,  r₂₆[0]]
+	VERIMG $32, R_1, T_0, R_1   // [_,  r²₂₆[1], _,  r₂₆[1]]
+	VERIMG $32, R_2, T_0, R_2   // [_,  r²₂₆[2], _,  r₂₆[2]]
+	VERIMG $32, R_3, T_0, R_3   // [_,  r²₂₆[3], _,  r₂₆[3]]
+	VERIMG $32, R_4, T_0, R_4   // [_,  r²₂₆[4], _,  r₂₆[4]]
+	VERIMG $32, R5_1, T_0, R5_1 // [_, 5r²₂₆[1], _, 5r₂₆[1]]
+	VERIMG $32, R5_2, T_0, R5_2 // [_, 5r²₂₆[2], _, 5r₂₆[2]]
+	VERIMG $32, R5_3, T_0, R5_3 // [_, 5r²₂₆[3], _, 5r₂₆[3]]
+	VERIMG $32, R5_4, T_0, R5_4 // [_, 5r²₂₆[4], _, 5r₂₆[4]]
 
 	MOVD $0, R3
 	BR   multiply
 
 skip:
-	VZERO H_0
-	VZERO H_1
-	VZERO H_2
-	VZERO H_3
-	VZERO H_4
-
 	CMPBEQ R3, $0, finish
 
-b1:
-	// 1 block remaining
-	SUB    $1, R3
-	VLL    R3, (R2), T_0
-	ADD    $1, R3
+b1:  // 1 block remaining
+
+	// Load the final block (1-16 bytes). This will be placed into
+	// lane 0.
+	MOVD $-1(R3), R0
+	VLL  R0, (R2), T_0 // pad to 16 bytes with zeros
+
+	// The Poly1305 algorithm requires that a 1 bit be appended to
+	// each message block. If the final block is less than 16 bytes
+	// long then it is easiest to insert the 1 before the message
+	// block is split into 26-bit limbs. If, on the other hand, the
+	// final message block is 16 bytes long then we append the 1 bit
+	// after expansion as normal.
 	MOVBZ  $1, R0
 	CMPBEQ R3, $16, 2(PC)
 	VLVGB  R3, R0, T_0
-	VZERO  T_1
-	EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4)
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $4, $1, F_4
-	VLEIG  $1, $1, R_0
-	VZERO  R_1
-	VZERO  R_2
-	VZERO  R_3
-	VZERO  R_4
-	VZERO  R5_1
-	VZERO  R5_2
-	VZERO  R5_3
-	VZERO  R5_4
 
-	// setup [r, 1]
-	VLVGG $0, RSAVE_0, R_0
-	VLVGG $0, RSAVE_1, R_1
-	VLVGG $0, RSAVE_2, R_2
-	VLVGG $0, RSAVE_3, R_3
-	VLVGG $0, RSAVE_4, R_4
-	VPDI  $0, R5SAVE_1, R5_1, R5_1
-	VPDI  $0, R5SAVE_2, R5_2, R5_2
-	VPDI  $0, R5SAVE_3, R5_3, R5_3
-	VPDI  $0, R5SAVE_4, R5_4, R5_4
+	// Set the message block in lane 1 to the value 0 so that it
+	// can be accumulated without affecting the final result.
+	VZERO T_1
+
+	// Split the final message block into 26-bit limbs in lane 0.
+	// Lane 1 will be contain 0.
+	EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4)
+
+	// Append a 1 byte to the end of the last block only if it is a
+	// full 16 byte block.
+	CMPBNE R3, $16, 2(PC)
+	VLEIB  $4, $1, M_4
+
+	// We have previously saved r and 5r in the 32-bit even indexes
+	// of the R_[0-4] and R5_[1-4] coefficient registers.
+	//
+	// We want lane 0 to be multiplied by r so we need to move the
+	// saved r value into the 32-bit odd index in lane 0. We want
+	// lane 1 to be set to the value 1. This makes multiplication
+	// a no-op. We do this by setting lane 1 in every register to 0
+	// and then just setting the 32-bit index 3 in R_0 to 1.
+	VZERO T_0
+	MOVD  $0, R0
+	MOVD  $0x10111213, R12
+	VLVGP R12, R0, T_1         // [_, 0x10111213, _, 0x00000000]
+	VPERM T_0, R_0, T_1, R_0   // [_,  r₂₆[0], _, 0]
+	VPERM T_0, R_1, T_1, R_1   // [_,  r₂₆[1], _, 0]
+	VPERM T_0, R_2, T_1, R_2   // [_,  r₂₆[2], _, 0]
+	VPERM T_0, R_3, T_1, R_3   // [_,  r₂₆[3], _, 0]
+	VPERM T_0, R_4, T_1, R_4   // [_,  r₂₆[4], _, 0]
+	VPERM T_0, R5_1, T_1, R5_1 // [_, 5r₂₆[1], _, 0]
+	VPERM T_0, R5_2, T_1, R5_2 // [_, 5r₂₆[2], _, 0]
+	VPERM T_0, R5_3, T_1, R5_3 // [_, 5r₂₆[3], _, 0]
+	VPERM T_0, R5_4, T_1, R5_4 // [_, 5r₂₆[4], _, 0]
+
+	// Set the value of lane 1 to be 1.
+	VLEIF $3, $1, R_0 // [_,  r₂₆[0], _, 1]
 
 	MOVD $0, R3
 	BR   multiply

poly1305/sum_vmsl_s390x.s

@@ -1,909 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,go1.11,!gccgo,!appengine
-
-#include "textflag.h"
-
-// Implementation of Poly1305 using the vector facility (vx) and the VMSL instruction.
-
-// constants
-#define EX0   V1
-#define EX1   V2
-#define EX2   V3
-
-// temporaries
-#define T_0 V4
-#define T_1 V5
-#define T_2 V6
-#define T_3 V7
-#define T_4 V8
-#define T_5 V9
-#define T_6 V10
-#define T_7 V11
-#define T_8 V12
-#define T_9 V13
-#define T_10 V14
-
-// r**2 & r**4
-#define R_0  V15
-#define R_1  V16
-#define R_2  V17
-#define R5_1 V18
-#define R5_2 V19
-// key (r)
-#define RSAVE_0 R7
-#define RSAVE_1 R8
-#define RSAVE_2 R9
-#define R5SAVE_1 R10
-#define R5SAVE_2 R11
-
-// message block
-#define M0 V20
-#define M1 V21
-#define M2 V22
-#define M3 V23
-#define M4 V24
-#define M5 V25
-
-// accumulator
-#define H0_0 V26
-#define H1_0 V27
-#define H2_0 V28
-#define H0_1 V29
-#define H1_1 V30
-#define H2_1 V31
-
-GLOBL ·keyMask<>(SB), RODATA, $16
-DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f
-DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f
-
-GLOBL ·bswapMask<>(SB), RODATA, $16
-DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908
-DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100
-
-GLOBL ·constants<>(SB), RODATA, $48
-// EX0
-DATA ·constants<>+0(SB)/8, $0x18191a1b1c1d1e1f
-DATA ·constants<>+8(SB)/8, $0x0000050403020100
-// EX1
-DATA ·constants<>+16(SB)/8, $0x18191a1b1c1d1e1f
-DATA ·constants<>+24(SB)/8, $0x00000a0908070605
-// EX2
-DATA ·constants<>+32(SB)/8, $0x18191a1b1c1d1e1f
-DATA ·constants<>+40(SB)/8, $0x0000000f0e0d0c0b
-
-GLOBL ·c<>(SB), RODATA, $48
-// EX0
-DATA ·c<>+0(SB)/8, $0x0000050403020100
-DATA ·c<>+8(SB)/8, $0x0000151413121110
-// EX1
-DATA ·c<>+16(SB)/8, $0x00000a0908070605
-DATA ·c<>+24(SB)/8, $0x00001a1918171615
-// EX2
-DATA ·c<>+32(SB)/8, $0x0000000f0e0d0c0b
-DATA ·c<>+40(SB)/8, $0x0000001f1e1d1c1b
-
-GLOBL ·reduce<>(SB), RODATA, $32
-// 44 bit
-DATA ·reduce<>+0(SB)/8, $0x0
-DATA ·reduce<>+8(SB)/8, $0xfffffffffff
-// 42 bit
-DATA ·reduce<>+16(SB)/8, $0x0
-DATA ·reduce<>+24(SB)/8, $0x3ffffffffff
-
-// h = (f*g) % (2**130-5) [partial reduction]
-// uses T_0...T_9 temporary registers
-// input: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2
-// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9
-// output: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2
-#define MULTIPLY(m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9) \
-	\ // Eliminate the dependency for the last 2 VMSLs
-	VMSLG m02_0, r_2, m4_2, m4_2                       \
-	VMSLG m13_0, r_2, m5_2, m5_2                       \ // 8 VMSLs pipelined
-	VMSLG m02_0, r_0, m4_0, m4_0                       \
-	VMSLG m02_1, r5_2, V0, T_0                         \
-	VMSLG m02_0, r_1, m4_1, m4_1                       \
-	VMSLG m02_1, r_0, V0, T_1                          \
-	VMSLG m02_1, r_1, V0, T_2                          \
-	VMSLG m02_2, r5_1, V0, T_3                         \
-	VMSLG m02_2, r5_2, V0, T_4                         \
-	VMSLG m13_0, r_0, m5_0, m5_0                       \
-	VMSLG m13_1, r5_2, V0, T_5                         \
-	VMSLG m13_0, r_1, m5_1, m5_1                       \
-	VMSLG m13_1, r_0, V0, T_6                          \
-	VMSLG m13_1, r_1, V0, T_7                          \
-	VMSLG m13_2, r5_1, V0, T_8                         \
-	VMSLG m13_2, r5_2, V0, T_9                         \
-	VMSLG m02_2, r_0, m4_2, m4_2                       \
-	VMSLG m13_2, r_0, m5_2, m5_2                       \
-	VAQ   m4_0, T_0, m02_0                             \
-	VAQ   m4_1, T_1, m02_1                             \
-	VAQ   m5_0, T_5, m13_0                             \
-	VAQ   m5_1, T_6, m13_1                             \
-	VAQ   m02_0, T_3, m02_0                            \
-	VAQ   m02_1, T_4, m02_1                            \
-	VAQ   m13_0, T_8, m13_0                            \
-	VAQ   m13_1, T_9, m13_1                            \
-	VAQ   m4_2, T_2, m02_2                             \
-	VAQ   m5_2, T_7, m13_2                             \
-
-// SQUARE uses three limbs of r and r_2*5 to output square of r
-// uses T_1, T_5 and T_7 temporary registers
-// input: r_0, r_1, r_2, r5_2
-// temp: TEMP0, TEMP1, TEMP2
-// output: p0, p1, p2
-#define SQUARE(r_0, r_1, r_2, r5_2, p0, p1, p2, TEMP0, TEMP1, TEMP2) \
-	VMSLG r_0, r_0, p0, p0     \
-	VMSLG r_1, r5_2, V0, TEMP0 \
-	VMSLG r_2, r5_2, p1, p1    \
-	VMSLG r_0, r_1, V0, TEMP1  \
-	VMSLG r_1, r_1, p2, p2     \
-	VMSLG r_0, r_2, V0, TEMP2  \
-	VAQ   TEMP0, p0, p0        \
-	VAQ   TEMP1, p1, p1        \
-	VAQ   TEMP2, p2, p2        \
-	VAQ   TEMP0, p0, p0        \
-	VAQ   TEMP1, p1, p1        \
-	VAQ   TEMP2, p2, p2        \
-
-// carry h0->h1->h2->h0 || h3->h4->h5->h3
-// uses T_2, T_4, T_5, T_7, T_8, T_9
-//       t6,  t7,  t8,  t9, t10, t11
-// input: h0, h1, h2, h3, h4, h5
-// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11
-// output: h0, h1, h2, h3, h4, h5
-#define REDUCE(h0, h1, h2, h3, h4, h5, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) \
-	VLM    (R12), t6, t7  \ // 44 and 42 bit clear mask
-	VLEIB  $7, $0x28, t10 \ // 5 byte shift mask
-	VREPIB $4, t8         \ // 4 bit shift mask
-	VREPIB $2, t11        \ // 2 bit shift mask
-	VSRLB  t10, h0, t0    \ // h0 byte shift
-	VSRLB  t10, h1, t1    \ // h1 byte shift
-	VSRLB  t10, h2, t2    \ // h2 byte shift
-	VSRLB  t10, h3, t3    \ // h3 byte shift
-	VSRLB  t10, h4, t4    \ // h4 byte shift
-	VSRLB  t10, h5, t5    \ // h5 byte shift
-	VSRL   t8, t0, t0     \ // h0 bit shift
-	VSRL   t8, t1, t1     \ // h2 bit shift
-	VSRL   t11, t2, t2    \ // h2 bit shift
-	VSRL   t8, t3, t3     \ // h3 bit shift
-	VSRL   t8, t4, t4     \ // h4 bit shift
-	VESLG  $2, t2, t9     \ // h2 carry x5
-	VSRL   t11, t5, t5    \ // h5 bit shift
-	VN     t6, h0, h0     \ // h0 clear carry
-	VAQ    t2, t9, t2     \ // h2 carry x5
-	VESLG  $2, t5, t9     \ // h5 carry x5
-	VN     t6, h1, h1     \ // h1 clear carry
-	VN     t7, h2, h2     \ // h2 clear carry
-	VAQ    t5, t9, t5     \ // h5 carry x5
-	VN     t6, h3, h3     \ // h3 clear carry
-	VN     t6, h4, h4     \ // h4 clear carry
-	VN     t7, h5, h5     \ // h5 clear carry
-	VAQ    t0, h1, h1     \ // h0->h1
-	VAQ    t3, h4, h4     \ // h3->h4
-	VAQ    t1, h2, h2     \ // h1->h2
-	VAQ    t4, h5, h5     \ // h4->h5
-	VAQ    t2, h0, h0     \ // h2->h0
-	VAQ    t5, h3, h3     \ // h5->h3
-	VREPG  $1, t6, t6     \ // 44 and 42 bit masks across both halves
-	VREPG  $1, t7, t7     \
-	VSLDB  $8, h0, h0, h0 \ // set up [h0/1/2, h3/4/5]
-	VSLDB  $8, h1, h1, h1 \
-	VSLDB  $8, h2, h2, h2 \
-	VO     h0, h3, h3     \
-	VO     h1, h4, h4     \
-	VO     h2, h5, h5     \
-	VESRLG $44, h3, t0    \ // 44 bit shift right
-	VESRLG $44, h4, t1    \
-	VESRLG $42, h5, t2    \
-	VN     t6, h3, h3     \ // clear carry bits
-	VN     t6, h4, h4     \
-	VN     t7, h5, h5     \
-	VESLG  $2, t2, t9     \ // multiply carry by 5
-	VAQ    t9, t2, t2     \
-	VAQ    t0, h4, h4     \
-	VAQ    t1, h5, h5     \
-	VAQ    t2, h3, h3     \
-
-// carry h0->h1->h2->h0
-// input: h0, h1, h2
-// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8
-// output: h0, h1, h2
-#define REDUCE2(h0, h1, h2, t0, t1, t2, t3, t4, t5, t6, t7, t8) \
-	VLEIB  $7, $0x28, t3 \ // 5 byte shift mask
-	VREPIB $4, t4        \ // 4 bit shift mask
-	VREPIB $2, t7        \ // 2 bit shift mask
-	VGBM   $0x003F, t5   \ // mask to clear carry bits
-	VSRLB  t3, h0, t0    \
-	VSRLB  t3, h1, t1    \
-	VSRLB  t3, h2, t2    \
-	VESRLG $4, t5, t5    \ // 44 bit clear mask
-	VSRL   t4, t0, t0    \
-	VSRL   t4, t1, t1    \
-	VSRL   t7, t2, t2    \
-	VESRLG $2, t5, t6    \ // 42 bit clear mask
-	VESLG  $2, t2, t8    \
-	VAQ    t8, t2, t2    \
-	VN     t5, h0, h0    \
-	VN     t5, h1, h1    \
-	VN     t6, h2, h2    \
-	VAQ    t0, h1, h1    \
-	VAQ    t1, h2, h2    \
-	VAQ    t2, h0, h0    \
-	VSRLB  t3, h0, t0    \
-	VSRLB  t3, h1, t1    \
-	VSRLB  t3, h2, t2    \
-	VSRL   t4, t0, t0    \
-	VSRL   t4, t1, t1    \
-	VSRL   t7, t2, t2    \
-	VN     t5, h0, h0    \
-	VN     t5, h1, h1    \
-	VESLG  $2, t2, t8    \
-	VN     t6, h2, h2    \
-	VAQ    t0, h1, h1    \
-	VAQ    t8, t2, t2    \
-	VAQ    t1, h2, h2    \
-	VAQ    t2, h0, h0    \
-
-// expands two message blocks into the lower halfs of the d registers
-// moves the contents of the d registers into upper halfs
-// input: in1, in2, d0, d1, d2, d3, d4, d5
-// temp: TEMP0, TEMP1, TEMP2, TEMP3
-// output: d0, d1, d2, d3, d4, d5
-#define EXPACC(in1, in2, d0, d1, d2, d3, d4, d5, TEMP0, TEMP1, TEMP2, TEMP3) \
-	VGBM   $0xff3f, TEMP0      \
-	VGBM   $0xff1f, TEMP1      \
-	VESLG  $4, d1, TEMP2       \
-	VESLG  $4, d4, TEMP3       \
-	VESRLG $4, TEMP0, TEMP0    \
-	VPERM  in1, d0, EX0, d0    \
-	VPERM  in2, d3, EX0, d3    \
-	VPERM  in1, d2, EX2, d2    \
-	VPERM  in2, d5, EX2, d5    \
-	VPERM  in1, TEMP2, EX1, d1 \
-	VPERM  in2, TEMP3, EX1, d4 \
-	VN     TEMP0, d0, d0       \
-	VN     TEMP0, d3, d3       \
-	VESRLG $4, d1, d1          \
-	VESRLG $4, d4, d4          \
-	VN     TEMP1, d2, d2       \
-	VN     TEMP1, d5, d5       \
-	VN     TEMP0, d1, d1       \
-	VN     TEMP0, d4, d4       \
-
-// expands one message block into the lower halfs of the d registers
-// moves the contents of the d registers into upper halfs
-// input: in, d0, d1, d2
-// temp: TEMP0, TEMP1, TEMP2
-// output: d0, d1, d2
-#define EXPACC2(in, d0, d1, d2, TEMP0, TEMP1, TEMP2) \
-	VGBM   $0xff3f, TEMP0     \
-	VESLG  $4, d1, TEMP2      \
-	VGBM   $0xff1f, TEMP1     \
-	VPERM  in, d0, EX0, d0    \
-	VESRLG $4, TEMP0, TEMP0   \
-	VPERM  in, d2, EX2, d2    \
-	VPERM  in, TEMP2, EX1, d1 \
-	VN     TEMP0, d0, d0      \
-	VN     TEMP1, d2, d2      \
-	VESRLG $4, d1, d1         \
-	VN     TEMP0, d1, d1      \
-
-// pack h2:h0 into h1:h0 (no carry)
-// input: h0, h1, h2
-// output: h0, h1, h2
-#define PACK(h0, h1, h2) \
-	VMRLG  h1, h2, h2  \ // copy h1 to upper half h2
-	VESLG  $44, h1, h1 \ // shift limb 1 44 bits, leaving 20
-	VO     h0, h1, h0  \ // combine h0 with 20 bits from limb 1
-	VESRLG $20, h2, h1 \ // put top 24 bits of limb 1 into h1
-	VLEIG  $1, $0, h1  \ // clear h2 stuff from lower half of h1
-	VO     h0, h1, h0  \ // h0 now has 88 bits (limb 0 and 1)
-	VLEIG  $0, $0, h2  \ // clear upper half of h2
-	VESRLG $40, h2, h1 \ // h1 now has upper two bits of result
-	VLEIB  $7, $88, h1 \ // for byte shift (11 bytes)
-	VSLB   h1, h2, h2  \ // shift h2 11 bytes to the left
-	VO     h0, h2, h0  \ // combine h0 with 20 bits from limb 1
-	VLEIG  $0, $0, h1  \ // clear upper half of h1
-
-// if h > 2**130-5 then h -= 2**130-5
-// input: h0, h1
-// temp: t0, t1, t2
-// output: h0
-#define MOD(h0, h1, t0, t1, t2) \
-	VZERO t0          \
-	VLEIG $1, $5, t0  \
-	VACCQ h0, t0, t1  \
-	VAQ   h0, t0, t0  \
-	VONE  t2          \
-	VLEIG $1, $-4, t2 \
-	VAQ   t2, t1, t1  \
-	VACCQ h1, t1, t1  \
-	VONE  t2          \
-	VAQ   t2, t1, t1  \
-	VN    h0, t1, t2  \
-	VNC   t0, t1, t1  \
-	VO    t1, t2, h0  \
-
-// func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]key)
-TEXT ·poly1305vmsl(SB), $0-32
-	// This code processes 6 + up to 4 blocks (32 bytes) per iteration
-	// using the algorithm described in:
-	// NEON crypto, Daniel J. Bernstein & Peter Schwabe
-	// https://cryptojedi.org/papers/neoncrypto-20120320.pdf
-	// And as moddified for VMSL as described in
-	// Accelerating Poly1305 Cryptographic Message Authentication on the z14
-	// O'Farrell et al, CASCON 2017, p48-55
-	// https://ibm.ent.box.com/s/jf9gedj0e9d2vjctfyh186shaztavnht
-
-	LMG   out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key
-	VZERO V0                // c
-
-	// load EX0, EX1 and EX2
-	MOVD $·constants<>(SB), R5
-	VLM  (R5), EX0, EX2        // c
-
-	// setup r
-	VL    (R4), T_0
-	MOVD  $·keyMask<>(SB), R6
-	VL    (R6), T_1
-	VN    T_0, T_1, T_0
-	VZERO T_2                 // limbs for r
-	VZERO T_3
-	VZERO T_4
-	EXPACC2(T_0, T_2, T_3, T_4, T_1, T_5, T_7)
-
-	// T_2, T_3, T_4: [0, r]
-
-	// setup r*20
-	VLEIG $0, $0, T_0
-	VLEIG $1, $20, T_0       // T_0: [0, 20]
-	VZERO T_5
-	VZERO T_6
-	VMSLG T_0, T_3, T_5, T_5
-	VMSLG T_0, T_4, T_6, T_6
-
-	// store r for final block in GR
-	VLGVG $1, T_2, RSAVE_0  // c
-	VLGVG $1, T_3, RSAVE_1  // c
-	VLGVG $1, T_4, RSAVE_2  // c
-	VLGVG $1, T_5, R5SAVE_1 // c
-	VLGVG $1, T_6, R5SAVE_2 // c
-
-	// initialize h
-	VZERO H0_0
-	VZERO H1_0
-	VZERO H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	// initialize pointer for reduce constants
-	MOVD $·reduce<>(SB), R12
-
-	// calculate r**2 and 20*(r**2)
-	VZERO R_0
-	VZERO R_1
-	VZERO R_2
-	SQUARE(T_2, T_3, T_4, T_6, R_0, R_1, R_2, T_1, T_5, T_7)
-	REDUCE2(R_0, R_1, R_2, M0, M1, M2, M3, M4, R5_1, R5_2, M5, T_1)
-	VZERO R5_1
-	VZERO R5_2
-	VMSLG T_0, R_1, R5_1, R5_1
-	VMSLG T_0, R_2, R5_2, R5_2
-
-	// skip r**4 calculation if 3 blocks or less
-	CMPBLE R3, $48, b4
-
-	// calculate r**4 and 20*(r**4)
-	VZERO T_8
-	VZERO T_9
-	VZERO T_10
-	SQUARE(R_0, R_1, R_2, R5_2, T_8, T_9, T_10, T_1, T_5, T_7)
-	REDUCE2(T_8, T_9, T_10, M0, M1, M2, M3, M4, T_2, T_3, M5, T_1)
-	VZERO T_2
-	VZERO T_3
-	VMSLG T_0, T_9, T_2, T_2
-	VMSLG T_0, T_10, T_3, T_3
-
-	// put r**2 to the right and r**4 to the left of R_0, R_1, R_2
-	VSLDB $8, T_8, T_8, T_8
-	VSLDB $8, T_9, T_9, T_9
-	VSLDB $8, T_10, T_10, T_10
-	VSLDB $8, T_2, T_2, T_2
-	VSLDB $8, T_3, T_3, T_3
-
-	VO T_8, R_0, R_0
-	VO T_9, R_1, R_1
-	VO T_10, R_2, R_2
-	VO T_2, R5_1, R5_1
-	VO T_3, R5_2, R5_2
-
-	CMPBLE R3, $80, load // less than or equal to 5 blocks in message
-
-	// 6(or 5+1) blocks
-	SUB    $81, R3
-	VLM    (R2), M0, M4
-	VLL    R3, 80(R2), M5
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBGE R3, $16, 2(PC)
-	VLVGB  R3, R0, M5
-	MOVD   $96(R2), R2
-	EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
-	EXPACC(M2, M3, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
-	VLEIB  $2, $1, H2_0
-	VLEIB  $2, $1, H2_1
-	VLEIB  $10, $1, H2_0
-	VLEIB  $10, $1, H2_1
-
-	VZERO  M0
-	VZERO  M1
-	VZERO  M2
-	VZERO  M3
-	VZERO  T_4
-	VZERO  T_10
-	EXPACC(M4, M5, M0, M1, M2, M3, T_4, T_10, T_0, T_1, T_2, T_3)
-	VLR    T_4, M4
-	VLEIB  $10, $1, M2
-	CMPBLT R3, $16, 2(PC)
-	VLEIB  $10, $1, T_10
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-
-	SUB    $16, R3
-	CMPBLE R3, $0, square
-
-load:
-	// load EX0, EX1 and EX2
-	MOVD $·c<>(SB), R5
-	VLM  (R5), EX0, EX2
-
-loop:
-	CMPBLE R3, $64, add // b4	// last 4 or less blocks left
-
-	// next 4 full blocks
-	VLM  (R2), M2, M5
-	SUB  $64, R3
-	MOVD $64(R2), R2
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, T_0, T_1, T_3, T_4, T_5, T_2, T_7, T_8, T_9)
-
-	// expacc in-lined to create [m2, m3] limbs
-	VGBM   $0x3f3f, T_0     // 44 bit clear mask
-	VGBM   $0x1f1f, T_1     // 40 bit clear mask
-	VPERM  M2, M3, EX0, T_3
-	VESRLG $4, T_0, T_0     // 44 bit clear mask ready
-	VPERM  M2, M3, EX1, T_4
-	VPERM  M2, M3, EX2, T_5
-	VN     T_0, T_3, T_3
-	VESRLG $4, T_4, T_4
-	VN     T_1, T_5, T_5
-	VN     T_0, T_4, T_4
-	VMRHG  H0_1, T_3, H0_0
-	VMRHG  H1_1, T_4, H1_0
-	VMRHG  H2_1, T_5, H2_0
-	VMRLG  H0_1, T_3, H0_1
-	VMRLG  H1_1, T_4, H1_1
-	VMRLG  H2_1, T_5, H2_1
-	VLEIB  $10, $1, H2_0
-	VLEIB  $10, $1, H2_1
-	VPERM  M4, M5, EX0, T_3
-	VPERM  M4, M5, EX1, T_4
-	VPERM  M4, M5, EX2, T_5
-	VN     T_0, T_3, T_3
-	VESRLG $4, T_4, T_4
-	VN     T_1, T_5, T_5
-	VN     T_0, T_4, T_4
-	VMRHG  V0, T_3, M0
-	VMRHG  V0, T_4, M1
-	VMRHG  V0, T_5, M2
-	VMRLG  V0, T_3, M3
-	VMRLG  V0, T_4, M4
-	VMRLG  V0, T_5, M5
-	VLEIB  $10, $1, M2
-	VLEIB  $10, $1, M5
-
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	CMPBNE R3, $0, loop
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-
-	// load EX0, EX1, EX2
-	MOVD $·constants<>(SB), R5
-	VLM  (R5), EX0, EX2
-
-	// sum vectors
-	VAQ H0_0, H0_1, H0_0
-	VAQ H1_0, H1_1, H1_0
-	VAQ H2_0, H2_1, H2_0
-
-	// h may be >= 2*(2**130-5) so we need to reduce it again
-	// M0...M4 are used as temps here
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-
-next:  // carry h1->h2
-	VLEIB  $7, $0x28, T_1
-	VREPIB $4, T_2
-	VGBM   $0x003F, T_3
-	VESRLG $4, T_3
-
-	// byte shift
-	VSRLB T_1, H1_0, T_4
-
-	// bit shift
-	VSRL T_2, T_4, T_4
-
-	// clear h1 carry bits
-	VN T_3, H1_0, H1_0
-
-	// add carry
-	VAQ T_4, H2_0, H2_0
-
-	// h is now < 2*(2**130-5)
-	// pack h into h1 (hi) and h0 (lo)
-	PACK(H0_0, H1_0, H2_0)
-
-	// if h > 2**130-5 then h -= 2**130-5
-	MOD(H0_0, H1_0, T_0, T_1, T_2)
-
-	// h += s
-	MOVD  $·bswapMask<>(SB), R5
-	VL    (R5), T_1
-	VL    16(R4), T_0
-	VPERM T_0, T_0, T_1, T_0    // reverse bytes (to big)
-	VAQ   T_0, H0_0, H0_0
-	VPERM H0_0, H0_0, T_1, H0_0 // reverse bytes (to little)
-	VST   H0_0, (R1)
-	RET
-
-add:
-	// load EX0, EX1, EX2
-	MOVD $·constants<>(SB), R5
-	VLM  (R5), EX0, EX2
-
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-	CMPBLE R3, $64, b4
-
-b4:
-	CMPBLE R3, $48, b3 // 3 blocks or less
-
-	// 4(3+1) blocks remaining
-	SUB    $49, R3
-	VLM    (R2), M0, M2
-	VLL    R3, 48(R2), M3
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M3
-	MOVD   $64(R2), R2
-	EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3)
-	VLEIB  $10, $1, H2_0
-	VLEIB  $10, $1, H2_1
-	VZERO  M0
-	VZERO  M1
-	VZERO  M4
-	VZERO  M5
-	VZERO  T_4
-	VZERO  T_10
-	EXPACC(M2, M3, M0, M1, M4, M5, T_4, T_10, T_0, T_1, T_2, T_3)
-	VLR    T_4, M2
-	VLEIB  $10, $1, M4
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, T_10
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M4, M5, M2, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG  V0, H0_1, H0_0
-	VMRHG  V0, H1_1, H1_0
-	VMRHG  V0, H2_1, H2_0
-	VMRLG  V0, H0_1, H0_1
-	VMRLG  V0, H1_1, H1_1
-	VMRLG  V0, H2_1, H2_1
-	SUB    $16, R3
-	CMPBLE R3, $0, square // this condition must always hold true!
-
-b3:
-	CMPBLE R3, $32, b2
-
-	// 3 blocks remaining
-
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// H*[r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, T_10, M5)
-
-	SUB    $33, R3
-	VLM    (R2), M0, M1
-	VLL    R3, 32(R2), M2
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M2
-
-	// H += m0
-	VZERO T_1
-	VZERO T_2
-	VZERO T_3
-	EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)
-	VLEIB $10, $1, T_3
-	VAG   H0_0, T_1, H0_0
-	VAG   H1_0, T_2, H1_0
-	VAG   H2_0, T_3, H2_0
-
-	VZERO M0
-	VZERO M3
-	VZERO M4
-	VZERO M5
-	VZERO T_10
-
-	// (H+m0)*r
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M3, M4, M5, V0, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_10, H0_1, H1_1, H2_1, T_9)
-
-	// H += m1
-	VZERO V0
-	VZERO T_1
-	VZERO T_2
-	VZERO T_3
-	EXPACC2(M1, T_1, T_2, T_3, T_4, T_5, T_6)
-	VLEIB $10, $1, T_3
-	VAQ   H0_0, T_1, H0_0
-	VAQ   H1_0, T_2, H1_0
-	VAQ   H2_0, T_3, H2_0
-	REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10)
-
-	// [H, m2] * [r**2, r]
-	EXPACC2(M2, H0_0, H1_0, H2_0, T_1, T_2, T_3)
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, H2_0
-	VZERO  M0
-	VZERO  M1
-	VZERO  M2
-	VZERO  M3
-	VZERO  M4
-	VZERO  M5
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, M5, T_10)
-	SUB    $16, R3
-	CMPBLE R3, $0, next   // this condition must always hold true!
-
-b2:
-	CMPBLE R3, $16, b1
-
-	// 2 blocks remaining
-
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// H*[r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9)
-	VMRHG V0, H0_1, H0_0
-	VMRHG V0, H1_1, H1_0
-	VMRHG V0, H2_1, H2_0
-	VMRLG V0, H0_1, H0_1
-	VMRLG V0, H1_1, H1_1
-	VMRLG V0, H2_1, H2_1
-
-	// move h to the left and 0s at the right
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-
-	// get message blocks and append 1 to start
-	SUB    $17, R3
-	VL     (R2), M0
-	VLL    R3, 16(R2), M1
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M1
-	VZERO  T_6
-	VZERO  T_7
-	VZERO  T_8
-	EXPACC2(M0, T_6, T_7, T_8, T_1, T_2, T_3)
-	EXPACC2(M1, T_6, T_7, T_8, T_1, T_2, T_3)
-	VLEIB  $2, $1, T_8
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, T_8
-
-	// add [m0, m1] to h
-	VAG H0_0, T_6, H0_0
-	VAG H1_0, T_7, H1_0
-	VAG H2_0, T_8, H2_0
-
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-	VZERO T_10
-	VZERO M0
-
-	// at this point R_0 .. R5_2 look like [r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M2, M3, M4, M5, T_10, M0, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M2, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10)
-	SUB    $16, R3, R3
-	CMPBLE R3, $0, next
-
-b1:
-	CMPBLE R3, $0, next
-
-	// 1 block remaining
-
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// H*[r**2, r]
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-
-	// set up [0, m0] limbs
-	SUB    $1, R3
-	VLL    R3, (R2), M0
-	ADD    $1, R3
-	MOVBZ  $1, R0
-	CMPBEQ R3, $16, 2(PC)
-	VLVGB  R3, R0, M0
-	VZERO  T_1
-	VZERO  T_2
-	VZERO  T_3
-	EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)// limbs: [0, m]
-	CMPBNE R3, $16, 2(PC)
-	VLEIB  $10, $1, T_3
-
-	// h+m0
-	VAQ H0_0, T_1, H0_0
-	VAQ H1_0, T_2, H1_0
-	VAQ H2_0, T_3, H2_0
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-
-	BR next
-
-square:
-	// setup [r²,r]
-	VSLDB $8, R_0, R_0, R_0
-	VSLDB $8, R_1, R_1, R_1
-	VSLDB $8, R_2, R_2, R_2
-	VSLDB $8, R5_1, R5_1, R5_1
-	VSLDB $8, R5_2, R5_2, R5_2
-
-	VLVGG $1, RSAVE_0, R_0
-	VLVGG $1, RSAVE_1, R_1
-	VLVGG $1, RSAVE_2, R_2
-	VLVGG $1, R5SAVE_1, R5_1
-	VLVGG $1, R5SAVE_2, R5_2
-
-	// setup [h0, h1]
-	VSLDB $8, H0_0, H0_0, H0_0
-	VSLDB $8, H1_0, H1_0, H1_0
-	VSLDB $8, H2_0, H2_0, H2_0
-	VO    H0_1, H0_0, H0_0
-	VO    H1_1, H1_0, H1_0
-	VO    H2_1, H2_0, H2_0
-	VZERO H0_1
-	VZERO H1_1
-	VZERO H2_1
-
-	VZERO M0
-	VZERO M1
-	VZERO M2
-	VZERO M3
-	VZERO M4
-	VZERO M5
-
-	// (h0*r**2) + (h1*r)
-	MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9)
-	REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5)
-	BR next

poly1305/vectors_test.go

@@ -2940,4 +2940,61 @@ var testData = [...]test{
 			"eeb979c28f03795f6f1e4b8410beab19a20febc91985b8a7c298534a6598" +
 			"f2c5b0dc5de9f5e55a97791507bc6373db26",
 	},
+
+	// Override initial state to ensure large h (subject to h < 2(2¹³⁰ - 5)) is
+	// deserialized from the state correctly.
+	{
+		key:   "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		state: "0000000000000007fffffffffffffffffffffffffffffff5", // 2(2¹³⁰ - 5) - 1
+		in:    "",
+		tag:   "f9ffffffffffffffffffffffffffffff",
+	},
+	{
+		key:   "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		state: "000000000000000700000000000000000000000000000000", // 2¹³⁰
+		in:    "",
+		tag:   "04000000000000000000000000000000",
+	},
+	{
+		key:   "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		state: "0000000000000007fffffffffffffffffffffffffffffff5", // 2(2¹³⁰ - 5) - 1
+		in: "ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff",
+		tag: "1b000e5e5dfe8f5c4da11dd17b7654e7",
+	},
+	{
+		key:   "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		state: "000000000000000700000000000000000000000000000001", // 2¹³⁰
+		in: "ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffffffffffffffffff",
+		tag: "380859a4a5860b0e0967edfd711d37de",
+	},
 }

sha3/sha3_test.go

@@ -17,6 +17,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"hash"
+	"math/rand"
 	"os"
 	"strings"
 	"testing"
@@ -306,8 +307,14 @@ func TestSqueezing(t *testing.T) {
 }
 
 // sequentialBytes produces a buffer of size consecutive bytes 0x00, 0x01, ..., used for testing.
+//
+// The alignment of each slice is intentionally randomized to detect alignment
+// issues in the implementation. See https://golang.org/issue/37644.
+// Ideally, the compiler should fuzz the alignment itself.
+// (See https://golang.org/issue/35128.)
 func sequentialBytes(size int) []byte {
-	result := make([]byte, size)
+	alignmentOffset := rand.Intn(8)
+	result := make([]byte, size+alignmentOffset)[alignmentOffset:]
 	for i := range result {
 		result[i] = byte(i)
 	}

sha3/xor_unaligned.go

@@ -16,6 +16,17 @@ func (b *storageBuf) asBytes() *[maxRate]byte {
 	return (*[maxRate]byte)(unsafe.Pointer(b))
 }
 
+//go:nocheckptr
+//
+// xorInUnaligned intentionally reads the input buffer as an unaligned slice of
+// integers. The language spec is not clear on whether that is allowed.
+// See:
+// 	https://golang.org/issue/37644
+// 	https://golang.org/issue/37298
+// 	https://golang.org/issue/35381
+
+// xorInUnaligned uses unaligned reads and writes to update d.a to contain d.a
+// XOR buf.
 func xorInUnaligned(d *state, buf []byte) {
 	n := len(buf)
 	bw := (*[maxRate / 8]uint64)(unsafe.Pointer(&buf[0]))[: n/8 : n/8]

ssh/agent/client.go

@@ -102,8 +102,9 @@ type ConstraintExtension struct {
 
 // AddedKey describes an SSH key to be added to an Agent.
 type AddedKey struct {
-	// PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey or
-	// *ecdsa.PrivateKey, which will be inserted into the agent.
+	// PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey,
+	// ed25519.PrivateKey or *ecdsa.PrivateKey, which will be inserted into the
+	// agent.
 	PrivateKey interface{}
 	// Certificate, if not nil, is communicated to the agent and will be
 	// stored with the key.
@@ -566,6 +567,17 @@ func (c *client) insertKey(s interface{}, comment string, constraints []byte) er
 			Comments:    comment,
 			Constraints: constraints,
 		})
+	case ed25519.PrivateKey:
+		req = ssh.Marshal(ed25519KeyMsg{
+			Type:        ssh.KeyAlgoED25519,
+			Pub:         []byte(k)[32:],
+			Priv:        []byte(k),
+			Comments:    comment,
+			Constraints: constraints,
+		})
+	// This function originally supported only *ed25519.PrivateKey, however the
+	// general idiom is to pass ed25519.PrivateKey by value, not by pointer.
+	// We still support the pointer variant for backwards compatibility.
 	case *ed25519.PrivateKey:
 		req = ssh.Marshal(ed25519KeyMsg{
 			Type:        ssh.KeyAlgoED25519,
@@ -683,6 +695,18 @@ func (c *client) insertCert(s interface{}, cert *ssh.Certificate, comment string
 			Comments:    comment,
 			Constraints: constraints,
 		})
+	case ed25519.PrivateKey:
+		req = ssh.Marshal(ed25519CertMsg{
+			Type:        cert.Type(),
+			CertBytes:   cert.Marshal(),
+			Pub:         []byte(k)[32:],
+			Priv:        []byte(k),
+			Comments:    comment,
+			Constraints: constraints,
+		})
+	// This function originally supported only *ed25519.PrivateKey, however the
+	// general idiom is to pass ed25519.PrivateKey by value, not by pointer.
+	// We still support the pointer variant for backwards compatibility.
 	case *ed25519.PrivateKey:
 		req = ssh.Marshal(ed25519CertMsg{
 			Type:        cert.Type(),

ssh/certs.go

@@ -414,8 +414,8 @@ func (c *CertChecker) CheckCert(principal string, cert *Certificate) error {
 	return nil
 }
 
-// SignCert sets c.SignatureKey to the authority's public key and stores a
-// Signature, by authority, in the certificate.
+// SignCert signs the certificate with an authority, setting the Nonce,
+// SignatureKey, and Signature fields.
 func (c *Certificate) SignCert(rand io.Reader, authority Signer) error {
 	c.Nonce = make([]byte, 32)
 	if _, err := io.ReadFull(rand, c.Nonce); err != nil {

ssh/cipher.go

@@ -119,7 +119,7 @@ var cipherModes = map[string]*cipherMode{
 	chacha20Poly1305ID: {64, 0, newChaCha20Cipher},
 
 	// CBC mode is insecure and so is not included in the default config.
-	// (See http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf). If absolutely
+	// (See https://www.ieee-security.org/TC/SP2013/papers/4977a526.pdf). If absolutely
 	// needed, it's possible to specify a custom Config to enable it.
 	// You should expect that an active attacker can recover plaintext if
 	// you do.

ssh/client_auth.go

@@ -36,7 +36,7 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error {
 
 	// during the authentication phase the client first attempts the "none" method
 	// then any untried methods suggested by the server.
-	tried := make(map[string]bool)
+	var tried []string
 	var lastMethods []string
 
 	sessionID := c.transport.getSessionID()
@@ -49,7 +49,9 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error {
 			// success
 			return nil
 		} else if ok == authFailure {
-			tried[auth.method()] = true
+			if m := auth.method(); !contains(tried, m) {
+				tried = append(tried, m)
+			}
 		}
 		if methods == nil {
 			methods = lastMethods
@@ -61,7 +63,7 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error {
 	findNext:
 		for _, a := range config.Auth {
 			candidateMethod := a.method()
-			if tried[candidateMethod] {
+			if contains(tried, candidateMethod) {
 				continue
 			}
 			for _, meth := range methods {
@@ -72,16 +74,16 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error {
 			}
 		}
 	}
-	return fmt.Errorf("ssh: unable to authenticate, attempted methods %v, no supported methods remain", keys(tried))
+	return fmt.Errorf("ssh: unable to authenticate, attempted methods %v, no supported methods remain", tried)
 }
 
-func keys(m map[string]bool) []string {
-	s := make([]string, 0, len(m))
-
-	for key := range m {
-		s = append(s, key)
+func contains(list []string, e string) bool {
+	for _, s := range list {
+		if s == e {
+			return true
+		}
 	}
-	return s
+	return false
 }
 
 // An AuthMethod represents an instance of an RFC 4252 authentication method.

ssh/internal/bcrypt_pbkdf/bcrypt_pbkdf.go

@@ -0,0 +1,93 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bcrypt_pbkdf implements bcrypt_pbkdf(3) from OpenBSD.
+//
+// See https://flak.tedunangst.com/post/bcrypt-pbkdf and
+// https://cvsweb.openbsd.org/cgi-bin/cvsweb/src/lib/libutil/bcrypt_pbkdf.c.
+package bcrypt_pbkdf
+
+import (
+	"crypto/sha512"
+	"errors"
+	"golang.org/x/crypto/blowfish"
+)
+
+const blockSize = 32
+
+// Key derives a key from the password, salt and rounds count, returning a
+// []byte of length keyLen that can be used as cryptographic key.
+func Key(password, salt []byte, rounds, keyLen int) ([]byte, error) {
+	if rounds < 1 {
+		return nil, errors.New("bcrypt_pbkdf: number of rounds is too small")
+	}
+	if len(password) == 0 {
+		return nil, errors.New("bcrypt_pbkdf: empty password")
+	}
+	if len(salt) == 0 || len(salt) > 1<<20 {
+		return nil, errors.New("bcrypt_pbkdf: bad salt length")
+	}
+	if keyLen > 1024 {
+		return nil, errors.New("bcrypt_pbkdf: keyLen is too large")
+	}
+
+	numBlocks := (keyLen + blockSize - 1) / blockSize
+	key := make([]byte, numBlocks*blockSize)
+
+	h := sha512.New()
+	h.Write(password)
+	shapass := h.Sum(nil)
+
+	shasalt := make([]byte, 0, sha512.Size)
+	cnt, tmp := make([]byte, 4), make([]byte, blockSize)
+	for block := 1; block <= numBlocks; block++ {
+		h.Reset()
+		h.Write(salt)
+		cnt[0] = byte(block >> 24)
+		cnt[1] = byte(block >> 16)
+		cnt[2] = byte(block >> 8)
+		cnt[3] = byte(block)
+		h.Write(cnt)
+		bcryptHash(tmp, shapass, h.Sum(shasalt))
+
+		out := make([]byte, blockSize)
+		copy(out, tmp)
+		for i := 2; i <= rounds; i++ {
+			h.Reset()
+			h.Write(tmp)
+			bcryptHash(tmp, shapass, h.Sum(shasalt))
+			for j := 0; j < len(out); j++ {
+				out[j] ^= tmp[j]
+			}
+		}
+
+		for i, v := range out {
+			key[i*numBlocks+(block-1)] = v
+		}
+	}
+	return key[:keyLen], nil
+}
+
+var magic = []byte("OxychromaticBlowfishSwatDynamite")
+
+func bcryptHash(out, shapass, shasalt []byte) {
+	c, err := blowfish.NewSaltedCipher(shapass, shasalt)
+	if err != nil {
+		panic(err)
+	}
+	for i := 0; i < 64; i++ {
+		blowfish.ExpandKey(shasalt, c)
+		blowfish.ExpandKey(shapass, c)
+	}
+	copy(out, magic)
+	for i := 0; i < 32; i += 8 {
+		for j := 0; j < 64; j++ {
+			c.Encrypt(out[i:i+8], out[i:i+8])
+		}
+	}
+	// Swap bytes due to different endianness.
+	for i := 0; i < 32; i += 4 {
+		out[i+3], out[i+2], out[i+1], out[i] = out[i], out[i+1], out[i+2], out[i+3]
+	}
+}

ssh/internal/bcrypt_pbkdf/bcrypt_pbkdf_test.go

@@ -0,0 +1,97 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bcrypt_pbkdf
+
+import (
+	"bytes"
+	"testing"
+)
+
+// Test vectors generated by the reference implementation from OpenBSD.
+var golden = []struct {
+	rounds                 int
+	password, salt, result []byte
+}{
+	{
+		12,
+		[]byte("password"),
+		[]byte("salt"),
+		[]byte{
+			0x1a, 0xe4, 0x2c, 0x05, 0xd4, 0x87, 0xbc, 0x02, 0xf6,
+			0x49, 0x21, 0xa4, 0xeb, 0xe4, 0xea, 0x93, 0xbc, 0xac,
+			0xfe, 0x13, 0x5f, 0xda, 0x99, 0x97, 0x4c, 0x06, 0xb7,
+			0xb0, 0x1f, 0xae, 0x14, 0x9a,
+		},
+	},
+	{
+		3,
+		[]byte("passwordy\x00PASSWORD\x00"),
+		[]byte("salty\x00SALT\x00"),
+		[]byte{
+			0x7f, 0x31, 0x0b, 0xd3, 0xe7, 0x8c, 0x32, 0x80, 0xc5,
+			0x9c, 0xe4, 0x59, 0x52, 0x11, 0xa2, 0x92, 0x8e, 0x8d,
+			0x4e, 0xc7, 0x44, 0xc1, 0xed, 0x2e, 0xfc, 0x9f, 0x76,
+			0x4e, 0x33, 0x88, 0xe0, 0xad,
+		},
+	},
+	{
+		// See http://thread.gmane.org/gmane.os.openbsd.bugs/20542
+		8,
+		[]byte("секретное слово"),
+		[]byte("посолить немножко"),
+		[]byte{
+			0x8d, 0xf4, 0x3f, 0xc6, 0xfe, 0x13, 0x1f, 0xc4, 0x7f,
+			0x0c, 0x9e, 0x39, 0x22, 0x4b, 0xd9, 0x4c, 0x70, 0xb6,
+			0xfc, 0xc8, 0xee, 0x81, 0x35, 0xfa, 0xdd, 0xf6, 0x11,
+			0x56, 0xe6, 0xcb, 0x27, 0x33, 0xea, 0x76, 0x5f, 0x31,
+			0x5a, 0x3e, 0x1e, 0x4a, 0xfc, 0x35, 0xbf, 0x86, 0x87,
+			0xd1, 0x89, 0x25, 0x4c, 0x1e, 0x05, 0xa6, 0xfe, 0x80,
+			0xc0, 0x61, 0x7f, 0x91, 0x83, 0xd6, 0x72, 0x60, 0xd6,
+			0xa1, 0x15, 0xc6, 0xc9, 0x4e, 0x36, 0x03, 0xe2, 0x30,
+			0x3f, 0xbb, 0x43, 0xa7, 0x6a, 0x64, 0x52, 0x3f, 0xfd,
+			0xa6, 0x86, 0xb1, 0xd4, 0x51, 0x85, 0x43,
+		},
+	},
+}
+
+func TestKey(t *testing.T) {
+	for i, v := range golden {
+		k, err := Key(v.password, v.salt, v.rounds, len(v.result))
+		if err != nil {
+			t.Errorf("%d: %s", i, err)
+			continue
+		}
+		if !bytes.Equal(k, v.result) {
+			t.Errorf("%d: expected\n%x\n, got\n%x\n", i, v.result, k)
+		}
+	}
+}
+
+func TestBcryptHash(t *testing.T) {
+	good := []byte{
+		0x87, 0x90, 0x48, 0x70, 0xee, 0xf9, 0xde, 0xdd, 0xf8, 0xe7,
+		0x61, 0x1a, 0x14, 0x01, 0x06, 0xe6, 0xaa, 0xf1, 0xa3, 0x63,
+		0xd9, 0xa2, 0xc5, 0x04, 0xdb, 0x35, 0x64, 0x43, 0x72, 0x1e,
+		0xb5, 0x55,
+	}
+	var pass, salt [64]byte
+	var result [32]byte
+	for i := 0; i < 64; i++ {
+		pass[i] = byte(i)
+		salt[i] = byte(i + 64)
+	}
+	bcryptHash(result[:], pass[:], salt[:])
+	if !bytes.Equal(result[:], good) {
+		t.Errorf("expected %x, got %x", good, result)
+	}
+}
+
+func BenchmarkKey(b *testing.B) {
+	pass := []byte("password")
+	salt := []byte("salt")
+	for i := 0; i < b.N; i++ {
+		Key(pass, salt, 10, 32)
+	}
+}

ssh/kex.go

@@ -572,7 +572,7 @@ func (gex *dhGEXSHA) diffieHellman(theirPublic, myPrivate *big.Int) (*big.Int, e
 	return new(big.Int).Exp(theirPublic, myPrivate, gex.p), nil
 }
 
-func (gex *dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshakeMagics) (*kexResult, error) {
+func (gex dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshakeMagics) (*kexResult, error) {
 	// Send GexRequest
 	kexDHGexRequest := kexDHGexRequestMsg{
 		MinBits:      dhGroupExchangeMinimumBits,
@@ -677,7 +677,7 @@ func (gex *dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshak
 // Server half implementation of the Diffie Hellman Key Exchange with SHA1 and SHA256.
 //
 // This is a minimal implementation to satisfy the automated tests.
-func (gex *dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshakeMagics, priv Signer) (result *kexResult, err error) {
+func (gex dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshakeMagics, priv Signer) (result *kexResult, err error) {
 	// Receive GexRequest
 	packet, err := c.readPacket()
 	if err != nil {

ssh/kex_test.go

@@ -9,9 +9,14 @@ package ssh
 import (
 	"crypto/rand"
 	"reflect"
+	"sync"
 	"testing"
 )
 
+// Runs multiple key exchanges concurrent to detect potential data races with
+// kex obtained from the global kexAlgoMap.
+// This test needs to be executed using the race detector in order to detect
+// race conditions.
 func TestKexes(t *testing.T) {
 	type kexResultErr struct {
 		result *kexResult
@@ -19,32 +24,42 @@ func TestKexes(t *testing.T) {
 	}
 
 	for name, kex := range kexAlgoMap {
-		a, b := memPipe()
+		t.Run(name, func(t *testing.T) {
+			wg := sync.WaitGroup{}
+			for i := 0; i < 3; i++ {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					a, b := memPipe()
 
-		s := make(chan kexResultErr, 1)
-		c := make(chan kexResultErr, 1)
-		var magics handshakeMagics
-		go func() {
-			r, e := kex.Client(a, rand.Reader, &magics)
-			a.Close()
-			c <- kexResultErr{r, e}
-		}()
-		go func() {
-			r, e := kex.Server(b, rand.Reader, &magics, testSigners["ecdsa"])
-			b.Close()
-			s <- kexResultErr{r, e}
-		}()
+					s := make(chan kexResultErr, 1)
+					c := make(chan kexResultErr, 1)
+					var magics handshakeMagics
+					go func() {
+						r, e := kex.Client(a, rand.Reader, &magics)
+						a.Close()
+						c <- kexResultErr{r, e}
+					}()
+					go func() {
+						r, e := kex.Server(b, rand.Reader, &magics, testSigners["ecdsa"])
+						b.Close()
+						s <- kexResultErr{r, e}
+					}()
 
-		clientRes := <-c
-		serverRes := <-s
-		if clientRes.err != nil {
-			t.Errorf("client: %v", clientRes.err)
-		}
-		if serverRes.err != nil {
-			t.Errorf("server: %v", serverRes.err)
-		}
-		if !reflect.DeepEqual(clientRes.result, serverRes.result) {
-			t.Errorf("kex %q: mismatch %#v, %#v", name, clientRes.result, serverRes.result)
-		}
+					clientRes := <-c
+					serverRes := <-s
+					if clientRes.err != nil {
+						t.Errorf("client: %v", clientRes.err)
+					}
+					if serverRes.err != nil {
+						t.Errorf("server: %v", serverRes.err)
+					}
+					if !reflect.DeepEqual(clientRes.result, serverRes.result) {
+						t.Errorf("kex %q: mismatch %#v, %#v", name, clientRes.result, serverRes.result)
+					}
+				}()
+			}
+			wg.Wait()
+		})
 	}
 }

ssh/keys.go

@@ -7,6 +7,8 @@ package ssh
 import (
 	"bytes"
 	"crypto"
+	"crypto/aes"
+	"crypto/cipher"
 	"crypto/dsa"
 	"crypto/ecdsa"
 	"crypto/elliptic"
@@ -25,6 +27,7 @@ import (
 	"strings"
 
 	"golang.org/x/crypto/ed25519"
+	"golang.org/x/crypto/ssh/internal/bcrypt_pbkdf"
 )
 
 // These constants represent the algorithm names for key types supported by this
@@ -559,9 +562,11 @@ func parseED25519(in []byte) (out PublicKey, rest []byte, err error) {
 		return nil, nil, err
 	}
 
-	key := ed25519.PublicKey(w.KeyBytes)
+	if l := len(w.KeyBytes); l != ed25519.PublicKeySize {
+		return nil, nil, fmt.Errorf("invalid size %d for Ed25519 public key", l)
+	}
 
-	return (ed25519PublicKey)(key), w.Rest, nil
+	return ed25519PublicKey(w.KeyBytes), w.Rest, nil
 }
 
 func (k ed25519PublicKey) Marshal() []byte {
@@ -579,9 +584,11 @@ func (k ed25519PublicKey) Verify(b []byte, sig *Signature) error {
 	if sig.Format != k.Type() {
 		return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, k.Type())
 	}
+	if l := len(k); l != ed25519.PublicKeySize {
+		return fmt.Errorf("ssh: invalid size %d for Ed25519 public key", l)
+	}
 
-	edKey := (ed25519.PublicKey)(k)
-	if ok := ed25519.Verify(edKey, b, sig.Blob); !ok {
+	if ok := ed25519.Verify(ed25519.PublicKey(k), b, sig.Blob); !ok {
 		return errors.New("ssh: signature did not verify")
 	}
 
@@ -835,6 +842,10 @@ func parseSKEd25519(in []byte) (out PublicKey, rest []byte, err error) {
 		return nil, nil, err
 	}
 
+	if l := len(w.KeyBytes); l != ed25519.PublicKeySize {
+		return nil, nil, fmt.Errorf("invalid size %d for Ed25519 public key", l)
+	}
+
 	key := new(skEd25519PublicKey)
 	key.application = w.Application
 	key.PublicKey = ed25519.PublicKey(w.KeyBytes)
@@ -859,6 +870,9 @@ func (k *skEd25519PublicKey) Verify(data []byte, sig *Signature) error {
 	if sig.Format != k.Type() {
 		return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, k.Type())
 	}
+	if l := len(k.PublicKey); l != ed25519.PublicKeySize {
+		return fmt.Errorf("invalid size %d for Ed25519 public key", l)
+	}
 
 	h := sha256.New()
 	h.Write([]byte(k.application))
@@ -895,8 +909,7 @@ func (k *skEd25519PublicKey) Verify(data []byte, sig *Signature) error {
 
 	original := Marshal(blob)
 
-	edKey := (ed25519.PublicKey)(k.PublicKey)
-	if ok := ed25519.Verify(edKey, original, edSig.Signature); !ok {
+	if ok := ed25519.Verify(k.PublicKey, original, edSig.Signature); !ok {
 		return errors.New("ssh: signature did not verify")
 	}
 
@@ -1048,7 +1061,10 @@ func NewPublicKey(key interface{}) (PublicKey, error) {
 	case *dsa.PublicKey:
 		return (*dsaPublicKey)(key), nil
 	case ed25519.PublicKey:
-		return (ed25519PublicKey)(key), nil
+		if l := len(key); l != ed25519.PublicKeySize {
+			return nil, fmt.Errorf("ssh: invalid size %d for Ed25519 public key", l)
+		}
+		return ed25519PublicKey(key), nil
 	default:
 		return nil, fmt.Errorf("ssh: unsupported key type %T", key)
 	}
@@ -1122,21 +1138,25 @@ func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 	case "DSA PRIVATE KEY":
 		return ParseDSAPrivateKey(block.Bytes)
 	case "OPENSSH PRIVATE KEY":
-		return parseOpenSSHPrivateKey(block.Bytes)
+		return parseOpenSSHPrivateKey(block.Bytes, unencryptedOpenSSHKey)
 	default:
 		return nil, fmt.Errorf("ssh: unsupported key type %q", block.Type)
 	}
 }
 
 // ParseRawPrivateKeyWithPassphrase returns a private key decrypted with
-// passphrase from a PEM encoded private key. If wrong passphrase, return
-// x509.IncorrectPasswordError.
+// passphrase from a PEM encoded private key. If the passphrase is wrong, it
+// will return x509.IncorrectPasswordError.
 func ParseRawPrivateKeyWithPassphrase(pemBytes, passphrase []byte) (interface{}, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil {
 		return nil, errors.New("ssh: no key found")
 	}
 
+	if block.Type == "OPENSSH PRIVATE KEY" {
+		return parseOpenSSHPrivateKey(block.Bytes, passphraseProtectedOpenSSHKey(passphrase))
+	}
+
 	if !encryptedBlock(block) || !x509.IsEncryptedPEMBlock(block) {
 		return nil, errors.New("ssh: not an encrypted key")
 	}
@@ -1193,9 +1213,68 @@ func ParseDSAPrivateKey(der []byte) (*dsa.PrivateKey, error) {
 	}, nil
 }
 
-// Implemented based on the documentation at
-// https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key
-func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) {
+func unencryptedOpenSSHKey(cipherName, kdfName, kdfOpts string, privKeyBlock []byte) ([]byte, error) {
+	if kdfName != "none" || cipherName != "none" {
+		return nil, &PassphraseMissingError{}
+	}
+	if kdfOpts != "" {
+		return nil, errors.New("ssh: invalid openssh private key")
+	}
+	return privKeyBlock, nil
+}
+
+func passphraseProtectedOpenSSHKey(passphrase []byte) openSSHDecryptFunc {
+	return func(cipherName, kdfName, kdfOpts string, privKeyBlock []byte) ([]byte, error) {
+		if kdfName == "none" || cipherName == "none" {
+			return nil, errors.New("ssh: key is not password protected")
+		}
+		if kdfName != "bcrypt" {
+			return nil, fmt.Errorf("ssh: unknown KDF %q, only supports %q", kdfName, "bcrypt")
+		}
+
+		var opts struct {
+			Salt   string
+			Rounds uint32
+		}
+		if err := Unmarshal([]byte(kdfOpts), &opts); err != nil {
+			return nil, err
+		}
+
+		k, err := bcrypt_pbkdf.Key(passphrase, []byte(opts.Salt), int(opts.Rounds), 32+16)
+		if err != nil {
+			return nil, err
+		}
+		key, iv := k[:32], k[32:]
+
+		c, err := aes.NewCipher(key)
+		if err != nil {
+			return nil, err
+		}
+		switch cipherName {
+		case "aes256-ctr":
+			ctr := cipher.NewCTR(c, iv)
+			ctr.XORKeyStream(privKeyBlock, privKeyBlock)
+		case "aes256-cbc":
+			if len(privKeyBlock)%c.BlockSize() != 0 {
+				return nil, fmt.Errorf("ssh: invalid encrypted private key length, not a multiple of the block size")
+			}
+			cbc := cipher.NewCBCDecrypter(c, iv)
+			cbc.CryptBlocks(privKeyBlock, privKeyBlock)
+		default:
+			return nil, fmt.Errorf("ssh: unknown cipher %q, only supports %q or %q", cipherName, "aes256-ctr", "aes256-cbc")
+		}
+
+		return privKeyBlock, nil
+	}
+}
+
+type openSSHDecryptFunc func(CipherName, KdfName, KdfOpts string, PrivKeyBlock []byte) ([]byte, error)
+
+// parseOpenSSHPrivateKey parses an OpenSSH private key, using the decrypt
+// function to unwrap the encrypted portion. unencryptedOpenSSHKey can be used
+// as the decrypt function to parse an unencrypted private key. See
+// https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key.
+func parseOpenSSHPrivateKey(key []byte, decrypt openSSHDecryptFunc) (crypto.PrivateKey, error) {
 	const magic = "openssh-key-v1\x00"
 	if len(key) < len(magic) || string(key[:len(magic)]) != magic {
 		return nil, errors.New("ssh: invalid openssh private key format")
@@ -1214,9 +1293,22 @@ func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) {
 	if err := Unmarshal(remaining, &w); err != nil {
 		return nil, err
 	}
+	if w.NumKeys != 1 {
+		// We only support single key files, and so does OpenSSH.
+		// https://github.com/openssh/openssh-portable/blob/4103a3ec7/sshkey.c#L4171
+		return nil, errors.New("ssh: multi-key files are not supported")
+	}
 
-	if w.KdfName != "none" || w.CipherName != "none" {
-		return nil, errors.New("ssh: cannot decode encrypted private keys")
+	privKeyBlock, err := decrypt(w.CipherName, w.KdfName, w.KdfOpts, w.PrivKeyBlock)
+	if err != nil {
+		if err, ok := err.(*PassphraseMissingError); ok {
+			pub, errPub := ParsePublicKey(w.PubKey)
+			if errPub != nil {
+				return nil, fmt.Errorf("ssh: failed to parse embedded public key: %v", errPub)
+			}
+			err.PublicKey = pub
+		}
+		return nil, err
 	}
 
 	pk1 := struct {
@@ -1226,15 +1318,13 @@ func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) {
 		Rest    []byte `ssh:"rest"`
 	}{}
 
-	if err := Unmarshal(w.PrivKeyBlock, &pk1); err != nil {
-		return nil, err
+	if err := Unmarshal(privKeyBlock, &pk1); err != nil || pk1.Check1 != pk1.Check2 {
+		if w.CipherName != "none" {
+			return nil, x509.IncorrectPasswordError
+		}
+		return nil, errors.New("ssh: malformed OpenSSH key")
 	}
 
-	if pk1.Check1 != pk1.Check2 {
-		return nil, errors.New("ssh: checkint mismatch")
-	}
-
-	// we only handle ed25519 and rsa keys currently
 	switch pk1.Keytype {
 	case KeyAlgoRSA:
 		// https://github.com/openssh/openssh-portable/blob/master/sshkey.c#L2760-L2773
@@ -1253,10 +1343,8 @@ func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) {
 			return nil, err
 		}
 
-		for i, b := range key.Pad {
-			if int(b) != i+1 {
-				return nil, errors.New("ssh: padding not as expected")
-			}
+		if err := checkOpenSSHKeyPadding(key.Pad); err != nil {
+			return nil, err
 		}
 
 		pk := &rsa.PrivateKey{
@@ -1291,20 +1379,78 @@ func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) {
 			return nil, errors.New("ssh: private key unexpected length")
 		}
 
-		for i, b := range key.Pad {
-			if int(b) != i+1 {
-				return nil, errors.New("ssh: padding not as expected")
-			}
+		if err := checkOpenSSHKeyPadding(key.Pad); err != nil {
+			return nil, err
 		}
 
 		pk := ed25519.PrivateKey(make([]byte, ed25519.PrivateKeySize))
 		copy(pk, key.Priv)
 		return &pk, nil
+	case KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521:
+		key := struct {
+			Curve   string
+			Pub     []byte
+			D       *big.Int
+			Comment string
+			Pad     []byte `ssh:"rest"`
+		}{}
+
+		if err := Unmarshal(pk1.Rest, &key); err != nil {
+			return nil, err
+		}
+
+		if err := checkOpenSSHKeyPadding(key.Pad); err != nil {
+			return nil, err
+		}
+
+		var curve elliptic.Curve
+		switch key.Curve {
+		case "nistp256":
+			curve = elliptic.P256()
+		case "nistp384":
+			curve = elliptic.P384()
+		case "nistp521":
+			curve = elliptic.P521()
+		default:
+			return nil, errors.New("ssh: unhandled elliptic curve: " + key.Curve)
+		}
+
+		X, Y := elliptic.Unmarshal(curve, key.Pub)
+		if X == nil || Y == nil {
+			return nil, errors.New("ssh: failed to unmarshal public key")
+		}
+
+		if key.D.Cmp(curve.Params().N) >= 0 {
+			return nil, errors.New("ssh: scalar is out of range")
+		}
+
+		x, y := curve.ScalarBaseMult(key.D.Bytes())
+		if x.Cmp(X) != 0 || y.Cmp(Y) != 0 {
+			return nil, errors.New("ssh: public key does not match private key")
+		}
+
+		return &ecdsa.PrivateKey{
+			PublicKey: ecdsa.PublicKey{
+				Curve: curve,
+				X:     X,
+				Y:     Y,
+			},
+			D: key.D,
+		}, nil
 	default:
 		return nil, errors.New("ssh: unhandled key type")
 	}
 }
 
+func checkOpenSSHKeyPadding(pad []byte) error {
+	for i, b := range pad {
+		if int(b) != i+1 {
+			return errors.New("ssh: padding not as expected")
+		}
+	}
+	return nil
+}
+
 // FingerprintLegacyMD5 returns the user presentation of the key's
 // fingerprint as described by RFC 4716 section 4.
 func FingerprintLegacyMD5(pubKey PublicKey) string {

ssh/mux.go

@@ -240,7 +240,7 @@ func (m *mux) onePacket() error {
 	id := binary.BigEndian.Uint32(packet[1:])
 	ch := m.chanList.getChan(id)
 	if ch == nil {
-		return fmt.Errorf("ssh: invalid channel %d", id)
+		return m.handleUnknownChannelPacket(id, packet)
 	}
 
 	return ch.handlePacket(packet)
@@ -328,3 +328,24 @@ func (m *mux) openChannel(chanType string, extra []byte) (*channel, error) {
 		return nil, fmt.Errorf("ssh: unexpected packet in response to channel open: %T", msg)
 	}
 }
+
+func (m *mux) handleUnknownChannelPacket(id uint32, packet []byte) error {
+	msg, err := decode(packet)
+	if err != nil {
+		return err
+	}
+
+	switch msg := msg.(type) {
+	// RFC 4254 section 5.4 says unrecognized channel requests should
+	// receive a failure response.
+	case *channelRequestMsg:
+		if msg.WantReply {
+			return m.sendMessage(channelRequestFailureMsg{
+				PeersID: msg.PeersID,
+			})
+		}
+		return nil
+	default:
+		return fmt.Errorf("ssh: invalid channel %d", id)
+	}
+}

ssh/mux_test.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"sync"
 	"testing"
+	"time"
 )
 
 func muxPair() (*mux, *mux) {
@@ -288,6 +289,219 @@ func TestMuxChannelRequest(t *testing.T) {
 	}
 }
 
+func TestMuxUnknownChannelRequests(t *testing.T) {
+	clientPipe, serverPipe := memPipe()
+	client := newMux(clientPipe)
+	defer serverPipe.Close()
+	defer client.Close()
+
+	kDone := make(chan struct{})
+	go func() {
+		// Ignore unknown channel messages that don't want a reply.
+		err := serverPipe.writePacket(Marshal(channelRequestMsg{
+			PeersID:             1,
+			Request:             "keepalive@openssh.com",
+			WantReply:           false,
+			RequestSpecificData: []byte{},
+		}))
+		if err != nil {
+			t.Fatalf("send: %v", err)
+		}
+
+		// Send a keepalive, which should get a channel failure message
+		// in response.
+		err = serverPipe.writePacket(Marshal(channelRequestMsg{
+			PeersID:             2,
+			Request:             "keepalive@openssh.com",
+			WantReply:           true,
+			RequestSpecificData: []byte{},
+		}))
+		if err != nil {
+			t.Fatalf("send: %v", err)
+		}
+
+		packet, err := serverPipe.readPacket()
+		if err != nil {
+			t.Fatalf("read packet: %v", err)
+		}
+		decoded, err := decode(packet)
+		if err != nil {
+			t.Fatalf("decode failed: %v", err)
+		}
+
+		switch msg := decoded.(type) {
+		case *channelRequestFailureMsg:
+			if msg.PeersID != 2 {
+				t.Fatalf("received response to wrong message: %v", msg)
+			}
+		default:
+			t.Fatalf("unexpected channel message: %v", msg)
+		}
+
+		kDone <- struct{}{}
+
+		// Receive and respond to the keepalive to confirm the mux is
+		// still processing requests.
+		packet, err = serverPipe.readPacket()
+		if err != nil {
+			t.Fatalf("read packet: %v", err)
+		}
+		if packet[0] != msgGlobalRequest {
+			t.Fatalf("expected global request")
+		}
+
+		err = serverPipe.writePacket(Marshal(globalRequestFailureMsg{
+			Data: []byte{},
+		}))
+		if err != nil {
+			t.Fatalf("failed to send failure msg: %v", err)
+		}
+
+		close(kDone)
+	}()
+
+	// Wait for the server to send the keepalive message and receive back a
+	// response.
+	select {
+	case <-kDone:
+	case <-time.After(10 * time.Second):
+		t.Fatalf("server never received ack")
+	}
+
+	// Confirm client hasn't closed.
+	if _, _, err := client.SendRequest("keepalive@golang.org", true, nil); err != nil {
+		t.Fatalf("failed to send keepalive: %v", err)
+	}
+
+	select {
+	case <-kDone:
+	case <-time.After(10 * time.Second):
+		t.Fatalf("server never shut down")
+	}
+}
+
+func TestMuxClosedChannel(t *testing.T) {
+	clientPipe, serverPipe := memPipe()
+	client := newMux(clientPipe)
+	defer serverPipe.Close()
+	defer client.Close()
+
+	kDone := make(chan struct{})
+	go func() {
+		// Open the channel.
+		packet, err := serverPipe.readPacket()
+		if err != nil {
+			t.Fatalf("read packet: %v", err)
+		}
+		if packet[0] != msgChannelOpen {
+			t.Fatalf("expected chan open")
+		}
+
+		var openMsg channelOpenMsg
+		if err := Unmarshal(packet, &openMsg); err != nil {
+			t.Fatalf("unmarshal: %v", err)
+		}
+
+		// Send back the opened channel confirmation.
+		err = serverPipe.writePacket(Marshal(channelOpenConfirmMsg{
+			PeersID:       openMsg.PeersID,
+			MyID:          0,
+			MyWindow:      0,
+			MaxPacketSize: channelMaxPacket,
+		}))
+		if err != nil {
+			t.Fatalf("send: %v", err)
+		}
+
+		// Close the channel.
+		err = serverPipe.writePacket(Marshal(channelCloseMsg{
+			PeersID: openMsg.PeersID,
+		}))
+		if err != nil {
+			t.Fatalf("send: %v", err)
+		}
+
+		// Send a keepalive message on the channel we just closed.
+		err = serverPipe.writePacket(Marshal(channelRequestMsg{
+			PeersID:             openMsg.PeersID,
+			Request:             "keepalive@openssh.com",
+			WantReply:           true,
+			RequestSpecificData: []byte{},
+		}))
+		if err != nil {
+			t.Fatalf("send: %v", err)
+		}
+
+		// Receive the channel closed response.
+		packet, err = serverPipe.readPacket()
+		if err != nil {
+			t.Fatalf("read packet: %v", err)
+		}
+		if packet[0] != msgChannelClose {
+			t.Fatalf("expected channel close")
+		}
+
+		// Receive the keepalive response failure.
+		packet, err = serverPipe.readPacket()
+		if err != nil {
+			t.Fatalf("read packet: %v", err)
+		}
+		if packet[0] != msgChannelFailure {
+			t.Fatalf("expected channel close")
+		}
+		kDone <- struct{}{}
+
+		// Receive and respond to the keepalive to confirm the mux is
+		// still processing requests.
+		packet, err = serverPipe.readPacket()
+		if err != nil {
+			t.Fatalf("read packet: %v", err)
+		}
+		if packet[0] != msgGlobalRequest {
+			t.Fatalf("expected global request")
+		}
+
+		err = serverPipe.writePacket(Marshal(globalRequestFailureMsg{
+			Data: []byte{},
+		}))
+		if err != nil {
+			t.Fatalf("failed to send failure msg: %v", err)
+		}
+
+		close(kDone)
+	}()
+
+	// Open a channel.
+	ch, err := client.openChannel("chan", nil)
+	if err != nil {
+		t.Fatalf("OpenChannel: %v", err)
+	}
+	defer ch.Close()
+
+	// Wait for the server to close the channel and send the keepalive.
+	select {
+	case <-kDone:
+	case <-time.After(10 * time.Second):
+		t.Fatalf("server never received ack")
+	}
+
+	// Make sure the channel closed.
+	if _, ok := <-ch.incomingRequests; ok {
+		t.Fatalf("channel not closed")
+	}
+
+	// Confirm client hasn't closed
+	if _, _, err := client.SendRequest("keepalive@golang.org", true, nil); err != nil {
+		t.Fatalf("failed to send keepalive: %v", err)
+	}
+
+	select {
+	case <-kDone:
+	case <-time.After(10 * time.Second):
+		t.Fatalf("server never shut down")
+	}
+}
+
 func TestMuxGlobalRequest(t *testing.T) {
 	clientMux, serverMux := muxPair()
 	defer serverMux.Close()

ssh/terminal/terminal.go

@@ -7,6 +7,7 @@ package terminal
 import (
 	"bytes"
 	"io"
+	"runtime"
 	"strconv"
 	"sync"
 	"unicode/utf8"
@@ -112,6 +113,7 @@ func NewTerminal(c io.ReadWriter, prompt string) *Terminal {
 }
 
 const (
+	keyCtrlC     = 3
 	keyCtrlD     = 4
 	keyCtrlU     = 21
 	keyEnter     = '\r'
@@ -150,8 +152,12 @@ func bytesToKey(b []byte, pasteActive bool) (rune, []byte) {
 		switch b[0] {
 		case 1: // ^A
 			return keyHome, b[1:]
+		case 2: // ^B
+			return keyLeft, b[1:]
 		case 5: // ^E
 			return keyEnd, b[1:]
+		case 6: // ^F
+			return keyRight, b[1:]
 		case 8: // ^H
 			return keyBackspace, b[1:]
 		case 11: // ^K
@@ -737,6 +743,9 @@ func (t *Terminal) readLine() (line string, err error) {
 						return "", io.EOF
 					}
 				}
+				if key == keyCtrlC {
+					return "", io.EOF
+				}
 				if key == keyPasteStart {
 					t.pasteActive = true
 					if len(t.line) == 0 {
@@ -939,6 +948,8 @@ func (s *stRingBuffer) NthPreviousEntry(n int) (value string, ok bool) {
 // readPasswordLine reads from reader until it finds \n or io.EOF.
 // The slice returned does not include the \n.
 // readPasswordLine also ignores any \r it finds.
+// Windows uses \r as end of line. So, on Windows, readPasswordLine
+// reads until it finds \r and ignores any \n it finds during processing.
 func readPasswordLine(reader io.Reader) ([]byte, error) {
 	var buf [1]byte
 	var ret []byte
@@ -952,9 +963,15 @@ func readPasswordLine(reader io.Reader) ([]byte, error) {
 					ret = ret[:len(ret)-1]
 				}
 			case '\n':
-				return ret, nil
+				if runtime.GOOS != "windows" {
+					return ret, nil
+				}
+				// otherwise ignore \n
 			case '\r':
-				// remove \r from passwords on Windows
+				if runtime.GOOS == "windows" {
+					return ret, nil
+				}
+				// otherwise ignore \r
 			default:
 				ret = append(ret, buf[0])
 			}

ssh/terminal/terminal_test.go

@@ -81,6 +81,14 @@ var keyPressTests = []struct {
 		in:   "a\x1b[Db\r", // left
 		line: "ba",
 	},
+	{
+		in:   "a\006b\r", // ^F
+		line: "ab",
+	},
+	{
+		in:   "a\002b\r", // ^B
+		line: "ba",
+	},
 	{
 		in:   "a\177b\r", // backspace
 		line: "b",
@@ -208,6 +216,16 @@ var keyPressTests = []struct {
 		line: "a",
 		err:  ErrPasteIndicator,
 	},
+	{
+		// Ctrl-C terminates readline
+		in:  "\003",
+		err: io.EOF,
+	},
+	{
+		// Ctrl-C at the end of line also terminates readline
+		in:  "a\003\r",
+		err: io.EOF,
+	},
 }
 
 func TestKeyPresses(t *testing.T) {
@@ -323,18 +341,32 @@ func TestTerminalSetSize(t *testing.T) {
 }
 
 func TestReadPasswordLineEnd(t *testing.T) {
-	var tests = []struct {
+	type testType struct {
 		input string
 		want  string
-	}{
-		{"\n", ""},
+	}
+	var tests = []testType{
 		{"\r\n", ""},
 		{"test\r\n", "test"},
+		{"test\r", "test"},
+		{"test\n", "test"},
 		{"testtesttesttes\n", "testtesttesttes"},
 		{"testtesttesttes\r\n", "testtesttesttes"},
 		{"testtesttesttesttest\n", "testtesttesttesttest"},
 		{"testtesttesttesttest\r\n", "testtesttesttesttest"},
+		{"\btest", "test"},
+		{"t\best", "est"},
+		{"te\bst", "tst"},
+		{"test\b", "tes"},
+		{"test\b\r\n", "tes"},
+		{"test\b\n", "tes"},
+		{"test\b\r", "tes"},
 	}
+	eol := "\n"
+	if runtime.GOOS == "windows" {
+		eol = "\r"
+	}
+	tests = append(tests, testType{eol, ""})
 	for _, test := range tests {
 		buf := new(bytes.Buffer)
 		if _, err := buf.WriteString(test.input); err != nil {

ssh/testdata/keys.go

@@ -113,6 +113,37 @@ pY2QA+P3QlrKxT/VWUMjHUbNNdYfJm48xu0SGNMRdKMAAABBAORh2NP/06JUV3J9W/2Hju
 X1ViJuqqcQnJPVzpgSL826EC2xwOECTqoY8uvFpUdD7CtpksIxNVqRIhuNOlz0lqEAAABB
 ANkaHTTaPojClO0dKJ/Zjs7pWOCGliebBYprQ/Y4r9QLBkC/XaWMS26gFIrjgC7D2Rv+rZ
 wSD0v0RcmkITP1ZR0AAAAYcHF1ZXJuYUBMdWNreUh5ZHJvLmxvY2FsAQID
+-----END OPENSSH PRIVATE KEY-----`),
+	"p256-openssh-format": []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAaAAAABNlY2RzYS
+1zaGEyLW5pc3RwMjU2AAAACG5pc3RwMjU2AAAAQQSN5Ld/DFy8LJK0yrWg+Ryhq4/ifHry
+QyCQeT4UXSB+UGdRct7kWA0hARbTaSCh+8U/Gs5O+IkDNoTKVsgxKUMQAAAAsO3C7nPtwu
+5zAAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBI3kt38MXLwskrTK
+taD5HKGrj+J8evJDIJB5PhRdIH5QZ1Fy3uRYDSEBFtNpIKH7xT8azk74iQM2hMpWyDEpQx
+AAAAAhAIHB48R+goZaiXndfYTrwk4BT1+MeLPC2/dwe0J5d1QDAAAAE21hcmlhbm9AZW5k
+b3IubG9jYWwBAgME
+-----END OPENSSH PRIVATE KEY-----`),
+	"p384-openssh-format": []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAiAAAABNlY2RzYS
+1zaGEyLW5pc3RwMzg0AAAACG5pc3RwMzg0AAAAYQTZb2VzEPs2NN/i1qHddKTVfwoIq3Tf
+PeQ/kcWBvuCVJfIygvpm9MeusawEPuLSEXwiNDew+YHZ9xHIvFjCmZsLuEOzuh9t9KotwM
+57H+7N+RDFzhM2j8hAaOuT5XDLKfUAAADgn/Sny5/0p8sAAAATZWNkc2Etc2hhMi1uaXN0
+cDM4NAAAAAhuaXN0cDM4NAAAAGEE2W9lcxD7NjTf4tah3XSk1X8KCKt03z3kP5HFgb7glS
+XyMoL6ZvTHrrGsBD7i0hF8IjQ3sPmB2fcRyLxYwpmbC7hDs7ofbfSqLcDOex/uzfkQxc4T
+No/IQGjrk+Vwyyn1AAAAMQDg0hwGKB/9Eq+e2FeTspi8QHW5xTD6prqsHDFx4cKk0ccgFV
+61dhFhD/8SEbYlHzEAAAATbWFyaWFub0BlbmRvci5sb2NhbAECAwQ=
+-----END OPENSSH PRIVATE KEY-----`),
+	"p521-openssh-format": []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS
+1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBKzI3QSp1a2e1zMulZl1uFF1Y2Dnv
+LSIwEu837hOV1epYEgNveAhGNm57TuBqYtnZeVfd2pzaz7CKX6N4B33N1XABQ5Ngji7lF2
+dUbmhNqJoMh43ioIsQNBaBenhmRpYP6f5k8P/7JZMIsLhkJk2hykb8maSZ+B3PYwPMNBdS
+vP+0sHQAAAEYIsr2CCLK9ggAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ
+AAAIUEASsyN0EqdWtntczLpWZdbhRdWNg57y0iMBLvN+4TldXqWBIDb3gIRjZue07gamLZ
+2XlX3dqc2s+wil+jeAd9zdVwAUOTYI4u5RdnVG5oTaiaDIeN4qCLEDQWgXp4ZkaWD+n+ZP
+D/+yWTCLC4ZCZNocpG/Jmkmfgdz2MDzDQXUrz/tLB0AAAAQgEdeH+im6iRcP/juTAoeSHo
+ExLtWhgL4JYqRwcOnzCKuLOPjEY/HSOuc+HRrbN9rbjsq+PcPHYe1NnkzXk0IW8hxQAAAB
+NtYXJpYW5vQGVuZG9yLmxvY2FsAQIDBAUGBw==
 -----END OPENSSH PRIVATE KEY-----`),
 	"user": []byte(`-----BEGIN EC PRIVATE KEY-----
 MHcCAQEEILYCAeq8f7V4vSSypRw7pxy8yz3V5W4qg8kSC3zJhqpQoAoGCCqGSM49
@@ -223,6 +254,36 @@ R/4X3HWfIdOHsDUvJ8xVWZ4BZk9Zk9qol045DcFCehpr/3hslCrKSZHakLt9GI58
 vVQJ4L0aYp5nloLfzhViZtKJXRLkySMKdzYkIlNmW1oVGl7tce5UCNI8Nok4j6yn
 IiHM7GBn+0nJoKTXsOGMIBe3ulKlKVxLjEuk9yivh/8=
 -----END DSA PRIVATE KEY-----
+`),
+	},
+
+	2: {
+		Name:              "ed25519-encrypted",
+		EncryptionKey:     "password",
+		IncludesPublicKey: true,
+		PEMBytes: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABDKj29BlC
+ocEWuVhQ94/RjoAAAAEAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAIIw1gSurPTDwZidA
+2AIjQZgoQi3IFn9jBtFdP10/Jj7DAAAAoFGkQbB2teSU7ikUsnc7ct2aH3pitM359lNVUh
+7DQbJWMjbQFbrBYyDJP+ALj1/RZmP2yoIf7/wr99q53/pm28Xp1gGP5V2RGRJYCA6kgFIH
+xdB6KEw1Ce7Bz8JaDIeagAGd3xtQTH3cuuleVxCZZnk9NspsPxigADKCls/RUiK7F+z3Qf
+Lvs9+PH8nIuhFMYZgo3liqZbVS5z4Fqhyzyq4=
+-----END OPENSSH PRIVATE KEY-----
+`),
+	},
+
+	3: {
+		Name:              "ed25519-encrypted-cbc",
+		EncryptionKey:     "password",
+		IncludesPublicKey: true,
+		PEMBytes: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jYmMAAAAGYmNyeXB0AAAAGAAAABDzGKF3uX
+G1gXALZKFd6Ir4AAAAEAAAAAEAAAAzAAAAC3NzaC1lZDI1NTE5AAAAIDne4/teO42zTDdj
+NwxUMNpbfmp/dxgU4ZNkC3ydgcugAAAAoJ3J/oA7+iqVOz0CIUUk9ufdP1VP4jDf2um+0s
+Sgs7x6Gpyjq67Ps7wLRdSmxr/G5b+Z8dRGFYS/wUCQEe3whwuImvLyPwWjXLzkAyMzc01f
+ywBGSrHnvP82ppenc2HuTI+E05Xc02i6JVyI1ShiekQL5twoqtR6pEBZnD17UonIx7cRzZ
+gbDGyT3bXMQtagvCwoW+/oMTKXiZP5jCJpEO8=
+-----END OPENSSH PRIVATE KEY-----
 `),
 	},
 }