go.crypto/ssh/terminal: support Unicode entry.

Previously, terminal only supported ASCII characters. This change
alters some []byte to []rune so that the full range of Unicode is
supported. The only thing that doesn't appear to work correctly are
grapheme clusters as the code still assumes one rune per glyph. Still,
this change allows many more languages to work than did previously.

R=golang-dev, rsc
CC=golang-dev
https://golang.org/cl/13704043

ssh/terminal/terminal.go

@@ -7,6 +7,7 @@ package terminal
 import (
 	"io"
 	"sync"
+	"unicode/utf8"
 )
 
 // EscapeCodes contains escape sequences that can be written to the terminal in
@@ -35,11 +36,12 @@ var vt100EscapeCodes = EscapeCodes{
 // Terminal contains the state for running a VT100 terminal that is capable of
 // reading lines of input.
 type Terminal struct {
-	// AutoCompleteCallback, if non-null, is called for each keypress
+	// AutoCompleteCallback, if non-null, is called for each keypress with
-	// with the full input line and the current position of the cursor.
+	// the full input line and the current position of the cursor (in
-	// If it returns a nil newLine, the key press is processed normally.
+	// bytes, as an index into |line|). If it returns ok=false, the key
-	// Otherwise it returns a replacement line and the new cursor position.
+	// press is processed normally. Otherwise it returns a replacement line
-	AutoCompleteCallback func(line []byte, pos, key int) (newLine []byte, newPos int)
+	// and the new cursor position.
+	AutoCompleteCallback func(line string, pos int, key rune) (newLine string, newPos int, ok bool)
 
 	// Escape contains a pointer to the escape codes for this terminal.
 	// It's always a valid pointer, although the escape codes themselves
@@ -54,7 +56,7 @@ type Terminal struct {
 	prompt string
 
 	// line is the current line being entered.
-	line []byte
+	line []rune
 	// pos is the logical position of the cursor in line
 	pos int
 	// echo is true if local echo is enabled
@@ -109,7 +111,7 @@ const (
 	keyEnter     = '\r'
 	keyEscape    = 27
 	keyBackspace = 127
-	keyUnknown   = 256 + iota
+	keyUnknown   = 0xd800 /* UTF-16 surrogate area */ + iota
 	keyUp
 	keyDown
 	keyLeft
@@ -123,10 +125,10 @@ const (
 )
 
 // bytesToKey tries to parse a key sequence from b. If successful, it returns
-// the key and the remainder of the input. Otherwise it returns -1.
+// the key and the remainder of the input. Otherwise it returns utf8.RuneError.
-func bytesToKey(b []byte) (int, []byte) {
+func bytesToKey(b []byte) (rune, []byte) {
 	if len(b) == 0 {
-		return -1, nil
+		return utf8.RuneError, nil
 	}
 
 	switch b[0] {
@@ -139,7 +141,11 @@ func bytesToKey(b []byte) (int, []byte) {
 	}
 
 	if b[0] != keyEscape {
-		return int(b[0]), b[1:]
+		if !utf8.FullRune(b) {
+			return utf8.RuneError, b
+		}
+		r, l := utf8.DecodeRune(b)
+		return r, b[l:]
 	}
 
 	if len(b) >= 3 && b[0] == keyEscape && b[1] == '[' {
@@ -183,19 +189,20 @@ func bytesToKey(b []byte) (int, []byte) {
 		}
 	}
 
-	return -1, b
+	return utf8.RuneError, b
 }
 
 // queue appends data to the end of t.outBuf
-func (t *Terminal) queue(data []byte) {
+func (t *Terminal) queue(data []rune) {
-	t.outBuf = append(t.outBuf, data...)
+	t.outBuf = append(t.outBuf, []byte(string(data))...)
 }
 
-var eraseUnderCursor = []byte{' ', keyEscape, '[', 'D'}
+var eraseUnderCursor = []rune{' ', keyEscape, '[', 'D'}
-var space = []byte{' '}
+var space = []rune{' '}
 
-func isPrintable(key int) bool {
+func isPrintable(key rune) bool {
-	return key >= 32 && key < 127
+	isInSurrogateArea := key >= 0xd800 && key <= 0xdbff
+	return key >= 32 && !isInSurrogateArea
 }
 
 // moveCursorToPos appends data to t.outBuf which will move the cursor to the
@@ -235,7 +242,7 @@ func (t *Terminal) moveCursorToPos(pos int) {
 }
 
 func (t *Terminal) move(up, down, left, right int) {
-	movement := make([]byte, 3*(up+down+left+right))
+	movement := make([]rune, 3*(up+down+left+right))
 	m := movement
 	for i := 0; i < up; i++ {
 		m[0] = keyEscape
@@ -266,13 +273,13 @@ func (t *Terminal) move(up, down, left, right int) {
 }
 
 func (t *Terminal) clearLineToRight() {
-	op := []byte{keyEscape, '[', 'K'}
+	op := []rune{keyEscape, '[', 'K'}
 	t.queue(op)
 }
 
 const maxLineLength = 4096
 
-func (t *Terminal) setLine(newLine []byte, newPos int) {
+func (t *Terminal) setLine(newLine []rune, newPos int) {
 	if t.echo {
 		t.moveCursorToPos(0)
 		t.writeLine(newLine)
@@ -354,7 +361,7 @@ func (t *Terminal) countToRightWord() int {
 
 // handleKey processes the given key and, optionally, returns a line of text
 // that the user has entered.
-func (t *Terminal) handleKey(key int) (line string, ok bool) {
+func (t *Terminal) handleKey(key rune) (line string, ok bool) {
 	switch key {
 	case keyBackspace:
 		if t.pos == 0 {
@@ -402,24 +409,24 @@ func (t *Terminal) handleKey(key int) (line string, ok bool) {
 			t.historyPending = string(t.line)
 		}
 		t.historyIndex++
-		t.setLine([]byte(entry), len(entry))
+		t.setLine([]rune(entry), len(entry))
 	case keyDown:
 		switch t.historyIndex {
 		case -1:
 			return
 		case 0:
-			t.setLine([]byte(t.historyPending), len(t.historyPending))
+			t.setLine([]rune(t.historyPending), len(t.historyPending))
 			t.historyIndex--
 		default:
 			entry, ok := t.history.NthPreviousEntry(t.historyIndex - 1)
 			if ok {
 				t.historyIndex--
-				t.setLine([]byte(entry), len(entry))
+				t.setLine([]rune(entry), len(entry))
 			}
 		}
 	case keyEnter:
 		t.moveCursorToPos(len(t.line))
-		t.queue([]byte("\r\n"))
+		t.queue([]rune("\r\n"))
 		line = string(t.line)
 		ok = true
 		t.line = t.line[:0]
@@ -441,12 +448,15 @@ func (t *Terminal) handleKey(key int) (line string, ok bool) {
 		t.moveCursorToPos(t.pos)
 	default:
 		if t.AutoCompleteCallback != nil {
+			prefix := string(t.line[:t.pos])
+			suffix := string(t.line[t.pos:])
+
 			t.lock.Unlock()
-			newLine, newPos := t.AutoCompleteCallback(t.line, t.pos, key)
+			newLine, newPos, completeOk := t.AutoCompleteCallback(prefix+suffix, len(prefix), key)
 			t.lock.Lock()
 
-			if newLine != nil {
+			if completeOk {
-				t.setLine(newLine, newPos)
+				t.setLine([]rune(newLine), utf8.RuneCount([]byte(newLine)[:newPos]))
 				return
 			}
 		}
@@ -457,13 +467,13 @@ func (t *Terminal) handleKey(key int) (line string, ok bool) {
 			return
 		}
 		if len(t.line) == cap(t.line) {
-			newLine := make([]byte, len(t.line), 2*(1+len(t.line)))
+			newLine := make([]rune, len(t.line), 2*(1+len(t.line)))
 			copy(newLine, t.line)
 			t.line = newLine
 		}
 		t.line = t.line[:len(t.line)+1]
 		copy(t.line[t.pos+1:], t.line[t.pos:])
-		t.line[t.pos] = byte(key)
+		t.line[t.pos] = key
 		if t.echo {
 			t.writeLine(t.line[t.pos:])
 		}
@@ -473,7 +483,7 @@ func (t *Terminal) handleKey(key int) (line string, ok bool) {
 	return
 }
 
-func (t *Terminal) writeLine(line []byte) {
+func (t *Terminal) writeLine(line []rune) {
 	for len(line) != 0 {
 		remainingOnLine := t.termWidth - t.cursorX
 		todo := len(line)
@@ -525,7 +535,7 @@ func (t *Terminal) Write(buf []byte) (n int, err error) {
 		return
 	}
 
-	t.queue([]byte(t.prompt))
+	t.queue([]rune(t.prompt))
 	chars := len(t.prompt)
 	if t.echo {
 		t.queue(t.line)
@@ -572,7 +582,7 @@ func (t *Terminal) readLine() (line string, err error) {
 	// t.lock must be held at this point
 
 	if t.cursorX == 0 && t.cursorY == 0 {
-		t.writeLine([]byte(t.prompt))
+		t.writeLine([]rune(t.prompt))
 		t.c.Write(t.outBuf)
 		t.outBuf = t.outBuf[:0]
 	}
@@ -581,9 +591,9 @@ func (t *Terminal) readLine() (line string, err error) {
 		rest := t.remainder
 		lineOk := false
 		for !lineOk {
-			var key int
+			var key rune
 			key, rest = bytesToKey(rest)
-			if key < 0 {
+			if key == utf8.RuneError {
 				break
 			}
 			if key == keyCtrlD {

ssh/terminal/terminal_test.go

@@ -137,6 +137,10 @@ var keyPressTests = []struct {
 		in:   "ab\x1b[D\013\r",
 		line: "a",
 	},
+	{
+		in:   "Ξεσκεπάζω\r",
+		line: "Ξεσκεπάζω",
+	},
 }
 
 func TestKeyPresses(t *testing.T) {