ssh: enforce strict limits on DSA key parameters

The parseDSA function previously accepted DSA keys with arbitrary values
for the sub-prime Q and did not validate that group elements G and Y
were within the modulus P.

Malicious actors could provide a key with a massively large Q (e.g.,
millions of bits), leading to excessive CPU consumption during signature
verification.

This change restricts the sub-prime Q to exactly 160 bits, as required
by FIPS 186-2, and ensures that G and Y are strictly less than P.

This issue was found during a security audit by NCC Group Cryptography
Services, sponsored by Teleport.

Fixes golang/go#79565
Fixes CVE-2026-39829

Change-Id: I526118d94684076088d0625178844f64c1303ec8
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/781661
Reviewed-by: Roland Shoemaker <roland@golang.org>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Neal Patel <nealpatel@google.com>

ssh/keys.go

@@ -580,6 +580,24 @@ func checkDSAParams(param *dsa.Parameters) error {
 		return fmt.Errorf("ssh: unsupported DSA key size %d", l)
 	}
 
+	// FIPS 186-2 specifies that Q must be exactly 160 bits. We must enforce
+	// this to prevent DoS attacks where an attacker sends a huge Q which makes
+	// verification slow.
+	if l := param.Q.BitLen(); l != 160 {
+		return fmt.Errorf("ssh: unsupported DSA sub-prime size %d", l)
+	}
+
+	// The generator G is an element of the group, so it must be strictly less
+	// than the modulus P.
+	if param.G.Cmp(param.P) >= 0 {
+		return errors.New("ssh: DSA generator larger than modulus")
+	}
+
+	// G must be positive.
+	if param.G.Sign() <= 0 {
+		return errors.New("ssh: DSA generator must be positive")
+	}
+
 	return nil
 }
 
@@ -602,6 +620,14 @@ func parseDSA(in []byte) (out PublicKey, rest []byte, err error) {
 		return nil, nil, err
 	}
 
+	// The public value Y must be a non-zero element of the group, i.e.
+	// strictly between 0 and P. crypto/dsa.Verify does not range-check Y,
+	// so we reject out-of-range values here to prevent a maliciously
+	// oversized Y from slowing verification.
+	if w.Y.Sign() <= 0 || w.Y.Cmp(w.P) >= 0 {
+		return nil, nil, errors.New("ssh: DSA public value Y out of range")
+	}
+
 	key := &dsaPublicKey{
 		Parameters: param,
 		Y:          w.Y,