ssh/knownhosts: a parser for the OpenSSH known_hosts file format

Change-Id: I271c90ff3a6d59e2e075c785a6bdb79e4b0849fa
Reviewed-on: https://go-review.googlesource.com/40354
Run-TryBot: Han-Wen Nienhuys <hanwen@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Adam Langley <agl@golang.org>

ssh/knownhosts/knownhosts.go

@@ -0,0 +1,469 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package knownhosts implements a parser for the OpenSSH
+// known_hosts host key database.
+package knownhosts
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// See the sshd manpage
+// (http://man.openbsd.org/sshd#SSH_KNOWN_HOSTS_FILE_FORMAT) for
+// background.
+
+type addr struct{ host, port string }
+
+func (a *addr) String() string {
+	return a.host + ":" + a.port
+}
+
+func (a *addr) eq(b addr) bool {
+	return a.host == b.host && a.port == b.port
+}
+
+type hostPattern struct {
+	negate bool
+	addr   addr
+}
+
+func (p *hostPattern) String() string {
+	n := ""
+	if p.negate {
+		n = "!"
+	}
+
+	return n + p.addr.String()
+}
+
+// See
+// https://android.googlesource.com/platform/external/openssh/+/ab28f5495c85297e7a597c1ba62e996416da7c7e/addrmatch.c
+// The matching of * has no regard for separators, unlike filesystem globs
+func wildcardMatch(pat []byte, str []byte) bool {
+	for {
+		if len(pat) == 0 {
+			return len(str) == 0
+		}
+		if len(str) == 0 {
+			return false
+		}
+
+		if pat[0] == '*' {
+			if len(pat) == 1 {
+				return true
+			}
+
+			for j := range str {
+				if wildcardMatch(pat[1:], str[j:]) {
+					return true
+				}
+			}
+			return false
+		}
+
+		if pat[0] == '?' || pat[0] == str[0] {
+			pat = pat[1:]
+			str = str[1:]
+		} else {
+			return false
+		}
+	}
+}
+
+func (l *hostPattern) match(a addr) bool {
+	return wildcardMatch([]byte(l.addr.host), []byte(a.host)) && l.addr.port == a.port
+}
+
+type keyDBLine struct {
+	cert     bool
+	patterns []*hostPattern
+	knownKey KnownKey
+}
+
+func (l *keyDBLine) String() string {
+	c := ""
+	if l.cert {
+		c = markerCert + " "
+	}
+
+	var ss []string
+	for _, p := range l.patterns {
+		ss = append(ss, p.String())
+	}
+
+	return c + strings.Join(ss, ",") + " " + serialize(l.knownKey.Key)
+}
+
+func serialize(k ssh.PublicKey) string {
+	return k.Type() + " " + base64.StdEncoding.EncodeToString(k.Marshal())
+}
+
+func (l *keyDBLine) match(addrs []addr) bool {
+	matched := false
+	for _, p := range l.patterns {
+		for _, a := range addrs {
+			m := p.match(a)
+			if p.negate {
+				if m {
+					return false
+				} else {
+					continue
+				}
+			}
+
+			if m {
+				matched = true
+			}
+		}
+	}
+
+	return matched
+}
+
+type hostKeyDB struct {
+	// Serialized version of revoked keys
+	revoked map[string]*KnownKey
+	lines   []keyDBLine
+}
+
+func (db *hostKeyDB) String() string {
+	var ls []string
+	for _, k := range db.revoked {
+		ls = append(ls, markerRevoked+" * "+serialize(k.Key))
+	}
+	for _, l := range db.lines {
+		ls = append(ls, l.String())
+	}
+	return strings.Join(ls, "\n")
+}
+
+func newHostKeyDB() *hostKeyDB {
+	db := &hostKeyDB{
+		revoked: make(map[string]*KnownKey),
+	}
+
+	return db
+}
+
+func keyEq(a, b ssh.PublicKey) bool {
+	return bytes.Equal(a.Marshal(), b.Marshal())
+}
+
+// IsAuthority can be used as a callback in ssh.CertChecker
+func (db *hostKeyDB) IsAuthority(remote ssh.PublicKey) bool {
+	for _, l := range db.lines {
+		// TODO(hanwen): should we check the hostname against host pattern?
+		if l.cert && keyEq(l.knownKey.Key, remote) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsRevoked can be used as a callback in ssh.CertChecker
+func (db *hostKeyDB) IsRevoked(key *ssh.Certificate) bool {
+	_, ok := db.revoked[string(key.Marshal())]
+	return ok
+}
+
+const markerCert = "@cert-authority"
+const markerRevoked = "@revoked"
+
+func nextWord(line []byte) (string, []byte) {
+	i := bytes.IndexAny(line, "\t ")
+	if i == -1 {
+		return string(line), nil
+	}
+
+	return string(line[:i]), bytes.TrimSpace(line[i:])
+}
+
+func parseLine(line []byte) (marker string, pattern []string, key ssh.PublicKey, err error) {
+	if w, next := nextWord(line); w == markerCert || w == markerRevoked {
+		marker = w
+		line = next
+	}
+
+	hostPart, line := nextWord(line)
+	if len(line) == 0 {
+		return "", nil, nil, errors.New("knownhosts: missing host pattern")
+	}
+
+	if len(hostPart) > 0 && hostPart[0] == '|' {
+		return "", nil, nil, errors.New("knownhosts: hashed hostnames not implemented")
+	}
+
+	pattern = strings.Split(hostPart, ",")
+
+	// ignore the keytype as it's in the key blob anyway.
+	_, line = nextWord(line)
+	if len(line) == 0 {
+		return "", nil, nil, errors.New("knownhosts: missing key type pattern")
+	}
+
+	keyBlob, _ := nextWord(line)
+
+	keyBytes, err := base64.StdEncoding.DecodeString(keyBlob)
+	if err != nil {
+		return "", nil, nil, err
+	}
+	key, err = ssh.ParsePublicKey(keyBytes)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	return marker, pattern, key, nil
+}
+
+func (db *hostKeyDB) parseLine(line []byte, filename string, linenum int) error {
+	marker, patterns, key, err := parseLine(line)
+	if err != nil {
+		return err
+	}
+
+	if marker == markerRevoked {
+		db.revoked[string(key.Marshal())] = &KnownKey{
+			Key:      key,
+			Filename: filename,
+			Line:     linenum,
+		}
+
+		return nil
+	}
+
+	entry := keyDBLine{
+		cert: marker == markerCert,
+		knownKey: KnownKey{
+			Filename: filename,
+			Line:     linenum,
+			Key:      key,
+		},
+	}
+
+	for _, p := range patterns {
+		if len(p) == 0 {
+			continue
+		}
+
+		var a addr
+		var negate bool
+		if p[0] == '!' {
+			negate = true
+			p = p[1:]
+		}
+
+		if len(p) == 0 {
+			return errors.New("knownhosts: negation without following hostname")
+		}
+
+		if p[0] == '[' {
+			a.host, a.port, err = net.SplitHostPort(p)
+			if err != nil {
+				return err
+			}
+		} else {
+			a.host, a.port, err = net.SplitHostPort(p)
+			if err != nil {
+				a.host = p
+				a.port = "22"
+			}
+		}
+
+		entry.patterns = append(entry.patterns, &hostPattern{
+			negate: negate,
+			addr:   a,
+		})
+	}
+
+	db.lines = append(db.lines, entry)
+	return nil
+}
+
+// KnownKey represents a key declared in a known_hosts file.
+type KnownKey struct {
+	Key      ssh.PublicKey
+	Filename string
+	Line     int
+}
+
+func (k *KnownKey) String() string {
+	return fmt.Sprintf("%s:%d: %s", k.Filename, k.Line, serialize(k.Key))
+}
+
+// KeyError is returned if we did not find the key in the host key
+// database, or there was a mismatch.  Typically, in batch
+// applications, this should be interpreted as failure. Interactive
+// applications can offer an interactive prompt to the user.
+type KeyError struct {
+	// Want holds the accepted host keys. For each key algorithm,
+	// there can be one hostkey.  If Want is empty, the host is
+	// unknown. If Want is non-empty, there was a mismatch, which
+	// can signify a MITM attack.
+	Want []KnownKey
+}
+
+func (u *KeyError) Error() string {
+	if len(u.Want) == 0 {
+		return "knownhosts: key is unknown"
+	}
+	return "knownhosts: key mismatch"
+}
+
+// RevokedError is returned if we found a key that was revoked.
+type RevokedError struct {
+	Revoked KnownKey
+}
+
+func (r *RevokedError) Error() string {
+	return "knownhosts: key is revoked"
+}
+
+// check checks a key against the host database. This should not be
+// used for verifying certificates.
+func (db *hostKeyDB) check(address string, remote net.Addr, remoteKey ssh.PublicKey) error {
+	if revoked := db.revoked[string(remoteKey.Marshal())]; revoked != nil {
+		return &RevokedError{Revoked: *revoked}
+	}
+
+	host, port, err := net.SplitHostPort(remote.String())
+	if err != nil {
+		return fmt.Errorf("knownhosts: SplitHostPort(%s): %v", remote, err)
+	}
+
+	addrs := []addr{
+		{host, port},
+	}
+
+	if address != "" {
+		host, port, err := net.SplitHostPort(address)
+		if err != nil {
+			return fmt.Errorf("knownhosts: SplitHostPort(%s): %v", address, err)
+		}
+
+		addrs = append(addrs, addr{host, port})
+	}
+
+	return db.checkAddrs(addrs, remoteKey)
+}
+
+// checkAddrs checks if we can find the given public key for any of
+// the given addresses.  If we only find an entry for the IP address,
+// or only the hostname, then this still succeeds.
+func (db *hostKeyDB) checkAddrs(addrs []addr, remoteKey ssh.PublicKey) error {
+	// TODO(hanwen): are these the right semantics? What if there
+	// is just a key for the IP address, but not for the
+	// hostname?
+
+	// Algorithm => key.
+	knownKeys := map[string]*KnownKey{}
+	for _, l := range db.lines {
+		if l.match(addrs) {
+			typ := l.knownKey.Key.Type()
+			if _, ok := knownKeys[typ]; !ok {
+				knownKeys[typ] = &l.knownKey
+			}
+		}
+	}
+
+	keyErr := &KeyError{}
+	for _, v := range knownKeys {
+		keyErr.Want = append(keyErr.Want, *v)
+	}
+
+	// Unknown remote host.
+	if len(knownKeys) == 0 {
+		return keyErr
+	}
+
+	// If the remote host starts using a different, unknown key type, we
+	// also interpret that as a mismatch.
+	if known := knownKeys[remoteKey.Type()]; known == nil || !keyEq(known.Key, remoteKey) {
+		return keyErr
+	}
+
+	return nil
+}
+
+// The Read function parses file contents.
+func (db *hostKeyDB) Read(r io.Reader, filename string) error {
+	scanner := bufio.NewScanner(r)
+
+	lineNum := 0
+	for scanner.Scan() {
+		lineNum++
+		line := scanner.Bytes()
+		line = bytes.TrimSpace(line)
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+
+		if err := db.parseLine(line, filename, lineNum); err != nil {
+			return err
+		}
+	}
+	return scanner.Err()
+}
+
+// New creates a host key callback from the given OpenSSH host key
+// files. The returned callback is for use in
+// ssh.ClientConfig.HostKeyCallback. Hostnames are ignored for
+// certificates, ie. any certificate authority is assumed to be valid
+// for all remote hosts.  Hashed hostnames are not supported.
+func New(files ...string) (ssh.HostKeyCallback, error) {
+	db := newHostKeyDB()
+	for _, fn := range files {
+		f, err := os.Open(fn)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		if err := db.Read(f, fn); err != nil {
+			return nil, err
+		}
+	}
+
+	// TODO(hanwen): properly supporting certificates requires an
+	// API change in the SSH library: IsAuthority should provide
+	// the address too?
+
+	var certChecker ssh.CertChecker
+	certChecker.IsAuthority = db.IsAuthority
+	certChecker.IsRevoked = db.IsRevoked
+	certChecker.HostKeyFallback = db.check
+
+	return certChecker.CheckHostKey, nil
+}
+
+// Line returns a line to add append to the known_hosts files.
+func Line(addresses []string, key ssh.PublicKey) string {
+	var trimmed []string
+	for _, a := range addresses {
+		host, port, err := net.SplitHostPort(a)
+		if err != nil {
+			host = a
+			port = "22"
+		}
+		entry := host
+		if port != "22" {
+			entry = "[" + entry + "]:" + port
+		} else if strings.Contains(host, ":") {
+			entry = "[" + entry + "]"
+		}
+
+		trimmed = append(trimmed, entry)
+	}
+
+	return strings.Join(trimmed, ",") + " " + serialize(key)
+}

ssh/knownhosts/knownhosts_test.go

@@ -0,0 +1,237 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package knownhosts
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+
+	"golang.org/x/crypto/ssh"
+)
+
+const edKeyStr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBAarftlLeoyf+v+nVchEZII/vna2PCV8FaX4vsF5BX"
+const alternateEdKeyStr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIXffBYeYL+WVzVru8npl5JHt2cjlr4ornFTWzoij9sx"
+const ecKeyStr = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNLCu01+wpXe3xB5olXCN4SqU2rQu0qjSRKJO4Bg+JRCPU+ENcgdA5srTU8xYDz/GEa4dzK5ldPw4J/gZgSXCMs="
+
+var ecKey, alternateEdKey, edKey ssh.PublicKey
+var testAddr = &net.TCPAddr{
+	IP:   net.IP{198, 41, 30, 196},
+	Port: 22,
+}
+
+var testAddr6 = &net.TCPAddr{
+	IP: net.IP{198, 41, 30, 196,
+		1, 2, 3, 4,
+		1, 2, 3, 4,
+		1, 2, 3, 4,
+	},
+	Port: 22,
+}
+
+func init() {
+	var err error
+	ecKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(ecKeyStr))
+	if err != nil {
+		panic(err)
+	}
+	edKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(edKeyStr))
+	if err != nil {
+		panic(err)
+	}
+	alternateEdKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(alternateEdKeyStr))
+	if err != nil {
+		panic(err)
+	}
+}
+
+func testDB(t *testing.T, s string) *hostKeyDB {
+	db := newHostKeyDB()
+	if err := db.Read(bytes.NewBufferString(s), "testdb"); err != nil {
+		t.Fatalf("Read: %v", err)
+	}
+
+	return db
+}
+
+func TestRevoked(t *testing.T) {
+	db := testDB(t, "\n\n@revoked * "+edKeyStr+"\n")
+	want := &RevokedError{
+		Revoked: KnownKey{
+			Key:      edKey,
+			Filename: "testdb",
+			Line:     3,
+		},
+	}
+	if err := db.check("", &net.TCPAddr{
+		Port: 42,
+	}, edKey); err == nil {
+		t.Fatal("no error for revoked key")
+	} else if !reflect.DeepEqual(want, err) {
+		t.Fatalf("got %#v, want %#v", want, err)
+	}
+}
+
+func TestBracket(t *testing.T) {
+	db := testDB(t, `[git.eclipse.org]:29418,[198.41.30.196]:29418 `+edKeyStr)
+
+	if err := db.check("git.eclipse.org:29418", &net.TCPAddr{
+		IP:   net.IP{198, 41, 30, 196},
+		Port: 29418,
+	}, edKey); err != nil {
+		t.Errorf("got error %v, want none", err)
+	}
+
+	if err := db.check("git.eclipse.org:29419", &net.TCPAddr{
+		Port: 42,
+	}, edKey); err == nil {
+		t.Fatalf("no error for unknown address")
+	} else if ke, ok := err.(*KeyError); !ok {
+		t.Fatalf("got type %T, want *KeyError", err)
+	} else if len(ke.Want) > 0 {
+		t.Fatalf("got Want %v, want []", ke.Want)
+	}
+}
+
+func TestNewKeyType(t *testing.T) {
+	str := fmt.Sprintf("%s %s", testAddr, edKeyStr)
+	db := testDB(t, str)
+	if err := db.check("", testAddr, ecKey); err == nil {
+		t.Fatalf("no error for unknown address")
+	} else if ke, ok := err.(*KeyError); !ok {
+		t.Fatalf("got type %T, want *KeyError", err)
+	} else if len(ke.Want) == 0 {
+		t.Fatalf("got empty KeyError.Want")
+	}
+}
+
+func TestSameKeyType(t *testing.T) {
+	str := fmt.Sprintf("%s %s", testAddr, edKeyStr)
+	db := testDB(t, str)
+	if err := db.check("", testAddr, alternateEdKey); err == nil {
+		t.Fatalf("no error for unknown address")
+	} else if ke, ok := err.(*KeyError); !ok {
+		t.Fatalf("got type %T, want *KeyError", err)
+	} else if len(ke.Want) == 0 {
+		t.Fatalf("got empty KeyError.Want")
+	} else if got, want := ke.Want[0].Key.Marshal(), edKey.Marshal(); !bytes.Equal(got, want) {
+		t.Fatalf("got key %q, want %q", got, want)
+	}
+}
+
+func TestIPAddress(t *testing.T) {
+	str := fmt.Sprintf("%s %s", testAddr, edKeyStr)
+	db := testDB(t, str)
+	if err := db.check("", testAddr, edKey); err != nil {
+		t.Errorf("got error %q, want none", err)
+	}
+}
+
+func TestIPv6Address(t *testing.T) {
+	str := fmt.Sprintf("%s %s", testAddr6, edKeyStr)
+	db := testDB(t, str)
+
+	if err := db.check("", testAddr6, edKey); err != nil {
+		t.Errorf("got error %q, want none", err)
+	}
+}
+
+func TestBasic(t *testing.T) {
+	str := fmt.Sprintf("#comment\n\nserver.org,%s %s", testAddr, edKeyStr)
+	db := testDB(t, str)
+	if err := db.check("server.org:22", testAddr, edKey); err != nil {
+		t.Errorf("got error %q, want none", err)
+	}
+
+	want := KnownKey{
+		Key:      edKey,
+		Filename: "testdb",
+		Line:     3,
+	}
+	if err := db.check("server.org:22", testAddr, ecKey); err == nil {
+		t.Errorf("succeeded, want KeyError")
+	} else if ke, ok := err.(*KeyError); !ok {
+		t.Errorf("got %T, want *KeyError", err)
+	} else if len(ke.Want) != 1 {
+		t.Errorf("got %v, want 1 entry", ke)
+	} else if !reflect.DeepEqual(ke.Want[0], want) {
+		t.Errorf("got %v, want %v", ke.Want[0], want)
+	}
+}
+
+func TestNegate(t *testing.T) {
+	str := fmt.Sprintf("%s,!server.org %s", testAddr, edKeyStr)
+	db := testDB(t, str)
+	if err := db.check("server.org:22", testAddr, ecKey); err == nil {
+		t.Errorf("succeeded")
+	} else if ke, ok := err.(*KeyError); !ok {
+		t.Errorf("got error type %T, want *KeyError", err)
+	} else if len(ke.Want) != 0 {
+		t.Errorf("got expected keys %d (first of type %s), want []", len(ke.Want), ke.Want[0].Key.Type())
+	}
+}
+
+func TestWildcard(t *testing.T) {
+	str := fmt.Sprintf("server*.domain %s", edKeyStr)
+	db := testDB(t, str)
+
+	want := &KeyError{
+		Want: []KnownKey{{
+			Filename: "testdb",
+			Line:     1,
+			Key:      edKey,
+		}},
+	}
+
+	got := db.check("server.domain:22", &net.TCPAddr{}, ecKey)
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %s, want %s", got, want)
+	}
+}
+
+func TestLine(t *testing.T) {
+	for in, want := range map[string]string{
+		"server.org":                             "server.org " + edKeyStr,
+		"server.org:22":                          "server.org " + edKeyStr,
+		"server.org:23":                          "[server.org]:23 " + edKeyStr,
+		"[c629:1ec4:102:304:102:304:102:304]:22": "[c629:1ec4:102:304:102:304:102:304] " + edKeyStr,
+		"[c629:1ec4:102:304:102:304:102:304]:23": "[c629:1ec4:102:304:102:304:102:304]:23 " + edKeyStr,
+	} {
+		if got := Line([]string{in}, edKey); got != want {
+			t.Errorf("Line(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestWildcardMatch(t *testing.T) {
+	for _, c := range []struct {
+		pat, str string
+		want     bool
+	}{
+		{"a?b", "abb", true},
+		{"ab", "abc", false},
+		{"abc", "ab", false},
+		{"a*b", "axxxb", true},
+		{"a*b", "axbxb", true},
+		{"a*b", "axbxbc", false},
+		{"a*?", "axbxc", true},
+		{"a*b*", "axxbxxxxxx", true},
+		{"a*b*c", "axxbxxxxxxc", true},
+		{"a*b*?", "axxbxxxxxxc", true},
+		{"a*b*z", "axxbxxbxxxz", true},
+		{"a*b*z", "axxbxxzxxxz", true},
+		{"a*b*z", "axxbxxzxxx", false},
+	} {
+		got := wildcardMatch([]byte(c.pat), []byte(c.str))
+		if got != c.want {
+			t.Errorf("wildcardMatch(%q, %q) = %v, want %v", c.pat, c.str, got, c.want)
+		}
+
+	}
+}
+
+// TODO(hanwen): test coverage for certificates.