ssh/agent: limit RSA key size on Add requests

parseRSAKey and parseRSACert build an rsa.PrivateKey directly from
the Add request body and then call priv.Precompute(). The CRT
coefficient recomputation in Precompute() is cubic in the size of
the prime factors, so accepting RSA keys with arbitrary modulus
and prime sizes can consume excessive CPU resources during a
single Add operation.

Add a checkRSAKeyParams helper that enforces the same bounds as
parseRSA in the ssh package (modulus <= 8192 bits, exponent <= 24
bits, exponent >= 3 and odd) plus a cap of 4096 bits on each prime
factor, and call it from both parseRSAKey and parseRSACert before
priv.Precompute().

The previous inline check on the exponent (BitLen > 30) is
subsumed by the new helper, which tightens it to BitLen > 24 for
consistency with parseRSA.

Change-Id: I6f0bbbfdc1db13d62a1f85e1d0e6ae517f000443
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/782421
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Reviewed-by: David Chase <drchase@google.com>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Junyang Shao <shaojunyang@google.com>

ssh/agent/server.go

@@ -240,13 +240,35 @@ func setConstraints(key *AddedKey, constraintBytes []byte) error {
 	return nil
 }
 
+// checkRSAKeyParams enforces the same bounds as parseRSA in the ssh
+// package, and additionally caps the prime factors. Without this,
+// the rsa.PrivateKey built from an Add request would call Precompute()
+// on arbitrary inputs; the CRT coefficient recomputation is cubic in
+// |p| and can consume excessive CPU on oversized keys.
+func checkRSAKeyParams(N, E, P, Q *big.Int) error {
+	if N.BitLen() > 8192 {
+		return errors.New("agent: RSA modulus too large")
+	}
+	if P.BitLen() > 4096 || Q.BitLen() > 4096 {
+		return errors.New("agent: RSA prime too large")
+	}
+	if E.BitLen() > 24 {
+		return errors.New("agent: RSA public exponent too large")
+	}
+	e := E.Int64()
+	if e < 3 || e&1 == 0 {
+		return errors.New("agent: incorrect RSA public exponent")
+	}
+	return nil
+}
+
 func parseRSAKey(req []byte) (*AddedKey, error) {
 	var k rsaKeyMsg
 	if err := ssh.Unmarshal(req, &k); err != nil {
 		return nil, err
 	}
-	if k.E.BitLen() > 30 {
-		return nil, errors.New("agent: RSA public exponent too large")
+	if err := checkRSAKeyParams(k.N, k.E, k.P, k.Q); err != nil {
+		return nil, err
 	}
 	priv := &rsa.PrivateKey{
 		PublicKey: rsa.PublicKey{
@@ -399,8 +421,8 @@ func parseRSACert(req []byte) (*AddedKey, error) {
 		return nil, fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err)
 	}
 
-	if rsaPub.E.BitLen() > 30 {
-		return nil, errors.New("agent: RSA public exponent too large")
+	if err := checkRSAKeyParams(rsaPub.N, rsaPub.E, k.P, k.Q); err != nil {
+		return nil, err
 	}
 
 	priv := rsa.PrivateKey{