New unicode-graphemes addon.

Handle grapheme cluster lookup as well as wcwidth.
This is based on Unicode 15, and could replace the unicode11 addon.

src/common/InputHandler.ts

@@ -10,13 +10,14 @@ import { CHARSETS, DEFAULT_CHARSET } from 'common/data/Charsets';
 import { EscapeSequenceParser } from 'common/parser/EscapeSequenceParser';
 import { Disposable } from 'common/Lifecycle';
 import { StringToUtf32, stringFromCodePoint, Utf8ToUtf32 } from 'common/input/TextDecoder';
-import { DEFAULT_ATTR_DATA } from 'common/buffer/BufferLine';
+import { BufferLine, DEFAULT_ATTR_DATA } from 'common/buffer/BufferLine';
 import { EventEmitter } from 'common/EventEmitter';
 import { IParsingState, IEscapeSequenceParser, IParams, IFunctionIdentifier } from 'common/parser/Types';
 import { NULL_CELL_CODE, NULL_CELL_WIDTH, Attributes, FgFlags, BgFlags, Content, UnderlineStyle } from 'common/buffer/Constants';
 import { CellData } from 'common/buffer/CellData';
 import { AttributeData } from 'common/buffer/AttributeData';
 import { ICoreService, IBufferService, IOptionsService, ILogService, ICoreMouseService, ICharsetService, IUnicodeService, LogLevelEnum, IOscLinkService } from 'common/services/Services';
+import { UnicodeService } from 'common/services/UnicodeService';
 import { OscHandler } from 'common/parser/OscParser';
 import { DcsHandler } from 'common/parser/DcsParser';
 import { IBuffer } from 'common/buffer/Types';
@@ -176,7 +177,7 @@ export class InputHandler extends Disposable implements IInputHandler {
     private readonly _oscLinkService: IOscLinkService,
     private readonly _coreMouseService: ICoreMouseService,
     private readonly _unicodeService: IUnicodeService,
-    private readonly _parser: IEscapeSequenceParser = new EscapeSequenceParser()
+    private readonly _parser: EscapeSequenceParser = new EscapeSequenceParser()
   ) {
     super();
     this.register(this._parser);
@@ -519,10 +520,6 @@ export class InputHandler extends Disposable implements IInputHandler {
     for (let pos = start; pos < end; ++pos) {
       code = data[pos];
 
-      // calculate print space
-      // expensive call, therefore we save width in line buffer
-      chWidth = this._unicodeService.wcwidth(code);
-
       // get charset replacement character
       // charset is only defined for ASCII, therefore we only
       // search for an replacement char if code < 127
@@ -533,6 +530,16 @@ export class InputHandler extends Disposable implements IInputHandler {
         }
       }
 
+      let precedingInfo = this._parser.precedingCodepoint === 0 ? 0
+        : this._parser.precedingJoinState;
+      // calculate print space
+      // expensive call, therefore we save width in line buffer
+      let currentInfo = this._unicodeService.charProperties(code, precedingInfo);
+      let chWidth = UnicodeService.extractWidth(currentInfo);
+      let shouldJoin = UnicodeService.extractShouldJoin(currentInfo);
+      const oldWidth = shouldJoin ? UnicodeService.extractWidth(precedingInfo) : 0;
+      this._parser.precedingCodepoint = code;
+      this._parser.precedingJoinState = currentInfo;
       if (screenReaderMode) {
         this._onA11yChar.fire(stringFromCodePoint(code));
       }
@@ -540,34 +547,16 @@ export class InputHandler extends Disposable implements IInputHandler {
         this._oscLinkService.addLineToLink(this._getCurrentLinkId(), this._activeBuffer.ybase + this._activeBuffer.y);
       }
 
-      // insert combining char at last cursor position
-      // this._activeBuffer.x should never be 0 for a combining char
-      // since they always follow a cell consuming char
-      // therefore we can test for this._activeBuffer.x to avoid overflow left
-      if (!chWidth && this._activeBuffer.x) {
-        if (!bufferRow.getWidth(this._activeBuffer.x - 1)) {
-          // found empty cell after fullwidth, need to go 2 cells back
-          // it is save to step 2 cells back here
-          // since an empty cell is only set by fullwidth chars
-          bufferRow.addCodepointToCell(this._activeBuffer.x - 2, code);
-        } else {
-          bufferRow.addCodepointToCell(this._activeBuffer.x - 1, code);
-        }
-        continue;
-      }
-
       // goto next line if ch would overflow
       // NOTE: To avoid costly width checks here,
       // the terminal does not allow a cols < 2.
-      if (this._activeBuffer.x + chWidth - 1 >= cols) {
+      if (this._activeBuffer.x + chWidth - oldWidth > cols) {
         // autowrap - DECAWM
         // automatically wraps to the beginning of the next line
         if (wraparoundMode) {
-          // clear left over cells to the right
-          while (this._activeBuffer.x < cols) {
-            bufferRow.setCellFromCodePoint(this._activeBuffer.x++, 0, 1, curAttr.fg, curAttr.bg, curAttr.extended);
-          }
-          this._activeBuffer.x = 0;
+          const oldRow = bufferRow;
+          let oldCol = this._activeBuffer.x - oldWidth;
+          this._activeBuffer.x = oldWidth;
           this._activeBuffer.y++;
           if (this._activeBuffer.y === this._activeBuffer.scrollBottom + 1) {
             this._activeBuffer.y--;
@@ -582,6 +571,16 @@ export class InputHandler extends Disposable implements IInputHandler {
           }
           // row changed, get it again
           bufferRow = this._activeBuffer.lines.get(this._activeBuffer.ybase + this._activeBuffer.y)!;
+          if (oldWidth > 0 && bufferRow instanceof BufferLine) {
+            // Combining character widens 1 column to 2.
+            // Move old character to next line.
+            bufferRow.copyCellsFrom(oldRow as BufferLine,
+                                    oldCol, 0, oldWidth, false);
+          }
+          // clear left over cells to the right
+          while (oldCol < cols) {
+            oldRow.setCellFromCodePoint(oldCol++, 0, 1, curAttr.fg, curAttr.bg, curAttr.extended);
+          }
         } else {
           this._activeBuffer.x = cols - 1;
           if (chWidth === 2) {
@@ -592,6 +591,21 @@ export class InputHandler extends Disposable implements IInputHandler {
         }
       }
 
+      // insert combining char at last cursor position
+      // this._activeBuffer.x should never be 0 for a combining char
+      // since they always follow a cell consuming char
+      // therefore we can test for this._activeBuffer.x to avoid overflow left
+      if (shouldJoin && this._activeBuffer.x) {
+        const offset = bufferRow.getWidth(this._activeBuffer.x - 1) ? 1 : 2
+        // if empty cell after fullwidth, need to go 2 cells back
+        // it is save to step 2 cells back here
+        // since an empty cell is only set by fullwidth chars
+        bufferRow.addCodepointToCell(this._activeBuffer.x - offset,
+                                     code, chWidth);
+        this._activeBuffer.x += chWidth - oldWidth;
+        continue;
+      }
+
       // insert mode: move characters to right
       if (insertMode) {
         // right shift cells according to the width
@@ -617,6 +631,7 @@ export class InputHandler extends Disposable implements IInputHandler {
         }
       }
     }
+      /*
     // store last char in Parser.precedingCodepoint for REP to work correctly
     // This needs to check whether:
     //  - fullwidth + surrogates: reset
@@ -631,7 +646,7 @@ export class InputHandler extends Disposable implements IInputHandler {
         this._parser.precedingCodepoint = this._workCell.content;
       }
     }
-
+      */
     // handle wide chars: reset cell to the right if it is second cell of a wide char
     if (this._activeBuffer.x < cols && end - start > 0 && bufferRow.getWidth(this._activeBuffer.x) === 0 && !bufferRow.hasContent(this._activeBuffer.x)) {
       bufferRow.setCellFromCodePoint(this._activeBuffer.x, 0, 1, curAttr.fg, curAttr.bg, curAttr.extended);