New unicode-graphemes addon.

Handle grapheme cluster lookup as well as wcwidth.
This is based on Unicode 15, and could replace the unicode11 addon.

src/common/InputHandler.ts

@@ -10,13 +10,14 @@ import { CHARSETS, DEFAULT_CHARSET } from 'common/data/Charsets';
 import { EscapeSequenceParser } from 'common/parser/EscapeSequenceParser';
 import { Disposable } from 'common/Lifecycle';
 import { StringToUtf32, stringFromCodePoint, Utf8ToUtf32 } from 'common/input/TextDecoder';
-import { DEFAULT_ATTR_DATA } from 'common/buffer/BufferLine';
+import { BufferLine, DEFAULT_ATTR_DATA } from 'common/buffer/BufferLine';
 import { EventEmitter } from 'common/EventEmitter';
 import { IParsingState, IEscapeSequenceParser, IParams, IFunctionIdentifier } from 'common/parser/Types';
 import { NULL_CELL_CODE, NULL_CELL_WIDTH, Attributes, FgFlags, BgFlags, Content, UnderlineStyle } from 'common/buffer/Constants';
 import { CellData } from 'common/buffer/CellData';
 import { AttributeData } from 'common/buffer/AttributeData';
 import { ICoreService, IBufferService, IOptionsService, ILogService, ICoreMouseService, ICharsetService, IUnicodeService, LogLevelEnum, IOscLinkService } from 'common/services/Services';
+import { UnicodeService } from 'common/services/UnicodeService';
 import { OscHandler } from 'common/parser/OscParser';
 import { DcsHandler } from 'common/parser/DcsParser';
 import { IBuffer } from 'common/buffer/Types';
@@ -176,7 +177,7 @@ export class InputHandler extends Disposable implements IInputHandler {
     private readonly _oscLinkService: IOscLinkService,
     private readonly _coreMouseService: ICoreMouseService,
     private readonly _unicodeService: IUnicodeService,
-    private readonly _parser: IEscapeSequenceParser = new EscapeSequenceParser()
+    private readonly _parser: EscapeSequenceParser = new EscapeSequenceParser()
   ) {
     super();
     this.register(this._parser);
@@ -519,10 +520,6 @@ export class InputHandler extends Disposable implements IInputHandler {
     for (let pos = start; pos < end; ++pos) {
       code = data[pos];
 
-      // calculate print space
-      // expensive call, therefore we save width in line buffer
-      chWidth = this._unicodeService.wcwidth(code);
-
       // get charset replacement character
       // charset is only defined for ASCII, therefore we only
       // search for an replacement char if code < 127
@@ -533,6 +530,16 @@ export class InputHandler extends Disposable implements IInputHandler {
         }
       }
 
+      let precedingInfo = this._parser.precedingCodepoint === 0 ? 0
+        : this._parser.precedingJoinState;
+      // calculate print space
+      // expensive call, therefore we save width in line buffer
+      let currentInfo = this._unicodeService.charProperties(code, precedingInfo);
+      let chWidth = UnicodeService.extractWidth(currentInfo);
+      let shouldJoin = UnicodeService.extractShouldJoin(currentInfo);
+      const oldWidth = shouldJoin ? UnicodeService.extractWidth(precedingInfo) : 0;
+      this._parser.precedingCodepoint = code;
+      this._parser.precedingJoinState = currentInfo;
       if (screenReaderMode) {
         this._onA11yChar.fire(stringFromCodePoint(code));
       }
@@ -540,34 +547,16 @@ export class InputHandler extends Disposable implements IInputHandler {
         this._oscLinkService.addLineToLink(this._getCurrentLinkId(), this._activeBuffer.ybase + this._activeBuffer.y);
       }
 
-      // insert combining char at last cursor position
-      // this._activeBuffer.x should never be 0 for a combining char
-      // since they always follow a cell consuming char
-      // therefore we can test for this._activeBuffer.x to avoid overflow left
-      if (!chWidth && this._activeBuffer.x) {
-        if (!bufferRow.getWidth(this._activeBuffer.x - 1)) {
-          // found empty cell after fullwidth, need to go 2 cells back
-          // it is save to step 2 cells back here
-          // since an empty cell is only set by fullwidth chars
-          bufferRow.addCodepointToCell(this._activeBuffer.x - 2, code);
-        } else {
-          bufferRow.addCodepointToCell(this._activeBuffer.x - 1, code);
-        }
-        continue;
-      }
-
       // goto next line if ch would overflow
       // NOTE: To avoid costly width checks here,
       // the terminal does not allow a cols < 2.
-      if (this._activeBuffer.x + chWidth - 1 >= cols) {
+      if (this._activeBuffer.x + chWidth - oldWidth > cols) {
         // autowrap - DECAWM
         // automatically wraps to the beginning of the next line
         if (wraparoundMode) {
-          // clear left over cells to the right
-          while (this._activeBuffer.x < cols) {
-            bufferRow.setCellFromCodePoint(this._activeBuffer.x++, 0, 1, curAttr.fg, curAttr.bg, curAttr.extended);
-          }
-          this._activeBuffer.x = 0;
+          const oldRow = bufferRow;
+          let oldCol = this._activeBuffer.x - oldWidth;
+          this._activeBuffer.x = oldWidth;
           this._activeBuffer.y++;
           if (this._activeBuffer.y === this._activeBuffer.scrollBottom + 1) {
             this._activeBuffer.y--;
@@ -582,6 +571,16 @@ export class InputHandler extends Disposable implements IInputHandler {
           }
           // row changed, get it again
           bufferRow = this._activeBuffer.lines.get(this._activeBuffer.ybase + this._activeBuffer.y)!;
+          if (oldWidth > 0 && bufferRow instanceof BufferLine) {
+            // Combining character widens 1 column to 2.
+            // Move old character to next line.
+            bufferRow.copyCellsFrom(oldRow as BufferLine,
+                                    oldCol, 0, oldWidth, false);
+          }
+          // clear left over cells to the right
+          while (oldCol < cols) {
+            oldRow.setCellFromCodePoint(oldCol++, 0, 1, curAttr.fg, curAttr.bg, curAttr.extended);
+          }
         } else {
           this._activeBuffer.x = cols - 1;
           if (chWidth === 2) {
@@ -592,6 +591,21 @@ export class InputHandler extends Disposable implements IInputHandler {
         }
       }
 
+      // insert combining char at last cursor position
+      // this._activeBuffer.x should never be 0 for a combining char
+      // since they always follow a cell consuming char
+      // therefore we can test for this._activeBuffer.x to avoid overflow left
+      if (shouldJoin && this._activeBuffer.x) {
+        const offset = bufferRow.getWidth(this._activeBuffer.x - 1) ? 1 : 2
+        // if empty cell after fullwidth, need to go 2 cells back
+        // it is save to step 2 cells back here
+        // since an empty cell is only set by fullwidth chars
+        bufferRow.addCodepointToCell(this._activeBuffer.x - offset,
+                                     code, chWidth);
+        this._activeBuffer.x += chWidth - oldWidth;
+        continue;
+      }
+
       // insert mode: move characters to right
       if (insertMode) {
         // right shift cells according to the width
@@ -617,6 +631,7 @@ export class InputHandler extends Disposable implements IInputHandler {
         }
       }
     }
+      /*
     // store last char in Parser.precedingCodepoint for REP to work correctly
     // This needs to check whether:
     //  - fullwidth + surrogates: reset
@@ -631,7 +646,7 @@ export class InputHandler extends Disposable implements IInputHandler {
         this._parser.precedingCodepoint = this._workCell.content;
       }
     }
-
+      */
     // handle wide chars: reset cell to the right if it is second cell of a wide char
     if (this._activeBuffer.x < cols && end - start > 0 && bufferRow.getWidth(this._activeBuffer.x) === 0 && !bufferRow.hasContent(this._activeBuffer.x)) {
       bufferRow.setCellFromCodePoint(this._activeBuffer.x, 0, 1, curAttr.fg, curAttr.bg, curAttr.extended);

src/common/TestUtils.test.ts

@@ -3,7 +3,8 @@
  * @license MIT
  */
 
-import { IBufferService, ICoreService, ILogService, IOptionsService, ITerminalOptions, ICoreMouseService, ICharsetService, IUnicodeService, IUnicodeVersionProvider, LogLevelEnum, IDecorationService, IInternalDecoration, IOscLinkService } from 'common/services/Services';
+import { IBufferService, ICoreService, ILogService, IOptionsService, ITerminalOptions, ICoreMouseService, ICharsetService, UnicodeCharProperties, UnicodeCharWidth, IUnicodeService, IUnicodeVersionProvider, LogLevelEnum, IDecorationService, IInternalDecoration, IOscLinkService } from 'common/services/Services';
+import { UnicodeService } from 'common/services/UnicodeService';
 import { IEvent, EventEmitter } from 'common/EventEmitter';
 import { clone } from 'common/Clone';
 import { DEFAULT_OPTIONS } from 'common/services/OptionsService';
@@ -167,7 +168,12 @@ export class MockUnicodeService implements IUnicodeService {
   public versions: string[] = [];
   public activeVersion: string = '';
   public onChange: IEvent<string> = new EventEmitter<string>().event;
-  public wcwidth = (codepoint: number): number => this._provider.wcwidth(codepoint);
+  public wcwidth = (codepoint: number): UnicodeCharWidth => this._provider.wcwidth(codepoint);
+  public charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
+    const w = this.wcwidth(codepoint);
+    const shouldJoin = w !== 0;
+    return UnicodeService.createPropertyValue(0, w, shouldJoin);
+  }
   public getStringCellWidth(s: string): number {
     throw new Error('Method not implemented.');
   }

src/common/Types.d.ts

@@ -231,7 +231,7 @@ export interface IBufferLine {
   loadCell(index: number, cell: ICellData): ICellData;
   setCell(index: number, cell: ICellData): void;
   setCellFromCodePoint(index: number, codePoint: number, width: number, fg: number, bg: number, eAttrs: IExtendedAttrs): void;
-  addCodepointToCell(index: number, codePoint: number): void;
+  addCodepointToCell(index: number, codePoint: number, width: number): void;
   insertCells(pos: number, n: number, ch: ICellData, eraseAttr?: IAttributeData): void;
   deleteCells(pos: number, n: number, fill: ICellData, eraseAttr?: IAttributeData): void;
   replaceCells(start: number, end: number, fill: ICellData, eraseAttr?: IAttributeData, respectProtect?: boolean): void;

src/common/buffer/BufferLine.test.ts

@@ -431,7 +431,7 @@ describe('BufferLine', function(): void {
   describe('addCharToCell', () => {
     it('should set width to 1 for empty cell', () => {
       const line = new TestBufferLine(3, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
-      line.addCodepointToCell(0, '\u0301'.charCodeAt(0));
+      line.addCodepointToCell(0, '\u0301'.charCodeAt(0), 0);
       const cell = line.loadCell(0, new CellData());
       // chars contains single combining char
       // width is set to 1
@@ -444,7 +444,7 @@ describe('BufferLine', function(): void {
       const cell = line .loadCell(0, new CellData());
       cell.setFromCharData([123, 'e\u0301', 1, 'e\u0301'.charCodeAt(1)]);
       line.setCell(0, cell);
-      line.addCodepointToCell(0, '\u0301'.charCodeAt(0));
+      line.addCodepointToCell(0, '\u0301'.charCodeAt(0), 0);
       line.loadCell(0, cell);
       // chars contains 3 chars
       // width is set to 1
@@ -457,7 +457,7 @@ describe('BufferLine', function(): void {
       const cell = line .loadCell(0, new CellData());
       cell.setFromCharData([123, 'e', 1, 'e'.charCodeAt(1)]);
       line.setCell(0, cell);
-      line.addCodepointToCell(0, '\u0301'.charCodeAt(0));
+      line.addCodepointToCell(0, '\u0301'.charCodeAt(0), 0);
       line.loadCell(0, cell);
       // chars contains 2 chars
       // width is set to 1

src/common/buffer/BufferLine.ts

@@ -227,7 +227,7 @@ export class BufferLine implements IBufferLine {
    * onto a leading char. Since we already set the attrs
    * by the previous `setDataFromCodePoint` call, we can omit it here.
    */
-  public addCodepointToCell(index: number, codePoint: number): void {
+    public addCodepointToCell(index: number, codePoint: number, width: number): void {
     let content = this._data[index * CELL_SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED_MASK) {
       // we already have a combined string, simply add
@@ -245,8 +245,12 @@ export class BufferLine implements IBufferLine {
         // simply set the data in the cell buffer with a width of 1
         content = codePoint | (1 << Content.WIDTH_SHIFT);
       }
-      this._data[index * CELL_SIZE + Cell.CONTENT] = content;
     }
+    if (width) {
+      content &= ~Content.WIDTH_MASK;
+      content |= width << Content.WIDTH_SHIFT;
+    }
+    this._data[index * CELL_SIZE + Cell.CONTENT] = content;
   }
 
   public insertCells(pos: number, n: number, fillCellData: ICellData, eraseAttr?: IAttributeData): void {

src/common/input/UnicodeV6.ts

@@ -3,8 +3,8 @@
  * @license MIT
  */
 import { IUnicodeVersionProvider } from 'common/services/Services';
-
-type CharWidth = 0 | 1 | 2;
+import { UnicodeCharProperties, UnicodeCharWidth } from 'common/services/Services';
+import { UnicodeService } from 'common/services/UnicodeService';
 
 const BMP_COMBINING = [
   [0x0300, 0x036F], [0x0483, 0x0486], [0x0488, 0x0489],
@@ -121,12 +121,28 @@ export class UnicodeV6 implements IUnicodeVersionProvider {
     }
   }
 
-  public wcwidth(num: number): CharWidth {
+  public wcwidth(num: number): UnicodeCharWidth {
     if (num < 32) return 0;
     if (num < 127) return 1;
-    if (num < 65536) return table[num] as CharWidth;
+    if (num < 65536) return table[num] as UnicodeCharWidth;
     if (bisearch(num, HIGH_COMBINING)) return 0;
     if ((num >= 0x20000 && num <= 0x2fffd) || (num >= 0x30000 && num <= 0x3fffd)) return 2;
     return 1;
   }
+
+  charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
+    let width = this.wcwidth(codepoint);
+    let shouldJoin = width === 0;
+    if (shouldJoin) {
+      let oldWidth = preceding === 0 ? 0
+        : UnicodeService.extractWidth(preceding);
+      if (oldWidth === 0) {
+        width = 1;
+        shouldJoin = false;
+      } else if (oldWidth > width) {
+        width = oldWidth;
+      }
+    }
+    return UnicodeService.createPropertyValue(0, width, shouldJoin);
+  }
 }

src/common/parser/EscapeSequenceParser.ts

@@ -231,6 +231,9 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
   public initialState: number;
   public currentState: number;
   public precedingCodepoint: number;
+  // Cached result of getJoinProperties(..., precedingCodepoint).
+  // Only valid if precedingCodepoint !== 0
+  public precedingJoinState: number = -1; // UnicodeJoinProperties
 
   // buffers over several parse calls
   protected _params: Params;

src/common/services/Services.ts

@@ -294,6 +294,11 @@ export interface IOscLinkService {
   getLinkData(linkId: number): IOscLinkData | undefined;
 }
 
+/** Width and Grapheme_Cluster_Break properties of a character. */
+export type UnicodeCharProperties = number;
+export const UnicodeInitialProperties: UnicodeCharProperties = 0; // UNEEDED?
+export type UnicodeCharWidth = 0 | 1 | 2;
+
 export const IUnicodeService = createDecorator<IUnicodeService>('UnicodeService');
 export interface IUnicodeService {
   serviceBrand: undefined;
@@ -309,13 +314,18 @@ export interface IUnicodeService {
   /**
    * Unicode version dependent
    */
-  wcwidth(codepoint: number): number;
+  wcwidth(codepoint: number): UnicodeCharWidth;
   getStringCellWidth(s: string): number;
+  /** Return character width, character type (for grapheme clustering).
+   * If preceding!=0, it is return code from previous character;
+   * in that case result specifies if characters should be joined. */
+  charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties;
 }
 
 export interface IUnicodeVersionProvider {
   readonly version: string;
-  wcwidth(ucs: number): 0 | 1 | 2;
+  wcwidth(ucs: number): UnicodeCharWidth;
+  charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties;
 }
 
 export const IDecorationService = createDecorator<IDecorationService>('DecorationService');

src/common/services/UnicodeService.test.ts

@@ -12,6 +12,9 @@ class DummyProvider implements IUnicodeVersionProvider {
   public wcwidth(n: number): 0 | 1 | 2 {
     return 2;
   }
+  charProperties(codepoint: number): number {
+    return UnicodeService.createPropertyValue(0, this.wcwidth(codepoint));
+  }
 }
 
 describe('unicode provider', () => {

src/common/services/UnicodeService.ts

@@ -2,7 +2,7 @@
  * Copyright (c) 2019 The xterm.js authors. All rights reserved.
  * @license MIT
  */
-import { IUnicodeService, IUnicodeVersionProvider } from 'common/services/Services';
+import { IUnicodeService, IUnicodeVersionProvider, UnicodeCharProperties, UnicodeCharWidth  } from 'common/services/Services';
 import { EventEmitter, IEvent } from 'common/EventEmitter';
 import { UnicodeV6 } from 'common/input/UnicodeV6';
 
@@ -16,6 +16,19 @@ export class UnicodeService implements IUnicodeService {
   private readonly _onChange = new EventEmitter<string>();
   public readonly onChange = this._onChange.event;
 
+  public static extractShouldJoin(value: UnicodeCharProperties): boolean {
+    return (value & 1) !== 0;
+  }
+  public static extractWidth(value: UnicodeCharProperties): UnicodeCharWidth {
+    return ((value >> 1) & 0x3) as UnicodeCharWidth;
+  }
+  public static extractCharKind(value: UnicodeCharProperties): number {
+    return value >> 3;
+  }
+  public static createPropertyValue(state: number, width: number, shouldJoin: boolean = false): UnicodeCharProperties {
+    return ((state & 0xffffff) << 3) | ((width & 3) << 1) | (shouldJoin?1:0);
+  }
+
   constructor() {
     const defaultProvider = new UnicodeV6();
     this.register(defaultProvider);
@@ -51,7 +64,7 @@ export class UnicodeService implements IUnicodeService {
   /**
    * Unicode version dependent interface.
    */
-  public wcwidth(num: number): number {
+  public wcwidth(num: number): UnicodeCharWidth {
     return this._activeProvider.wcwidth(num);
   }
 
@@ -83,4 +96,8 @@ export class UnicodeService implements IUnicodeService {
     }
     return result;
   }
+
+  charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
+    return this._activeProvider.charProperties(codepoint, preceding);
+  }
 }