ssh: support RSA SHA-2 (RFC8332) signatures

This change adds support for RSA SHA-2 based signatures for host keys and certificates. It also switches the default certificate signature algorithm for RSA to use SHA-512. This is implemented by treating ssh.Signer specially when the key type is `ssh-rsa` by also allowing SHA-256 and SHA-512 signatures.

Fixes golang/go#37278

Change-Id: I2ee1ac4ae4c9c1de441a2d6cf1e806357ef18910
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/220037
Trust: Jason A. Donenfeld <Jason@zx2c4.com>
Run-TryBot: Jason A. Donenfeld <Jason@zx2c4.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jason A. Donenfeld <Jason@zx2c4.com>
Reviewed-by: Roland Shoemaker <roland@golang.org>

ssh/client_test.go

@@ -5,6 +5,8 @@
 package ssh
 
 import (
+	"bytes"
+	"crypto/rand"
 	"strings"
 	"testing"
 )
@@ -116,6 +118,45 @@ func TestHostKeyCheck(t *testing.T) {
 	}
 }
 
+func TestVerifyHostKeySignature(t *testing.T) {
+	for _, tt := range []struct {
+		key        string
+		signAlgo   string
+		verifyAlgo string
+		wantError  string
+	}{
+		{"rsa", SigAlgoRSA, SigAlgoRSA, ""},
+		{"rsa", SigAlgoRSASHA2256, SigAlgoRSASHA2256, ""},
+		{"rsa", SigAlgoRSA, SigAlgoRSASHA2512, `ssh: invalid signature algorithm "ssh-rsa", expected "rsa-sha2-512"`},
+		{"ed25519", KeyAlgoED25519, KeyAlgoED25519, ""},
+	} {
+		key := testSigners[tt.key].PublicKey()
+		s, ok := testSigners[tt.key].(AlgorithmSigner)
+		if !ok {
+			t.Fatalf("needed an AlgorithmSigner")
+		}
+		sig, err := s.SignWithAlgorithm(rand.Reader, []byte("test"), tt.signAlgo)
+		if err != nil {
+			t.Fatalf("couldn't sign: %q", err)
+		}
+
+		b := bytes.Buffer{}
+		writeString(&b, []byte(sig.Format))
+		writeString(&b, sig.Blob)
+
+		result := kexResult{Signature: b.Bytes(), H: []byte("test")}
+
+		err = verifyHostKeySignature(key, tt.verifyAlgo, &result)
+		if err != nil {
+			if tt.wantError == "" || !strings.Contains(err.Error(), tt.wantError) {
+				t.Errorf("got error %q, expecting %q", err.Error(), tt.wantError)
+			}
+		} else if tt.wantError != "" {
+			t.Errorf("succeeded, but want error string %q", tt.wantError)
+		}
+	}
+}
+
 func TestBannerCallback(t *testing.T) {
 	c1, c2, err := netPipe()
 	if err != nil {