Merge pull request #5883 from vsych/fix-webgl-atlas-merge-stale-vertices

Fix webgl rendering corruption from atlas page merges

addons/addon-webgl/src/GlyphRenderer.ts

@@ -358,7 +358,9 @@ export class GlyphRenderer extends Disposable {
     gl.bindBuffer(gl.ARRAY_BUFFER, this._attributesBuffer);
     gl.bufferData(gl.ARRAY_BUFFER, activeBuffer.subarray(0, bufferLength), gl.STREAM_DRAW);
 
-    // Bind the atlas page texture if they have changed
+    // Bind the atlas page texture if they have changed. AtlasPage.version is globally
+    // monotonic, so a page object swap at the same index (which happens after a page merge)
+    // is detected by the same comparison.
     for (let i = 0; i < this._atlas.pages.length; i++) {
       if (this._atlas.pages[i].version !== this._atlasTextures[i].version) {
         this._bindAtlasPageTexture(gl, this._atlas, i);
@@ -371,6 +373,10 @@ export class GlyphRenderer extends Disposable {
 
   public setAtlas(atlas: ITextureAtlas): void {
     this._atlas = atlas;
+    this.invalidateAtlasTextures();
+  }
+
+  public invalidateAtlasTextures(): void {
     for (const glTexture of this._atlasTextures) {
       glTexture.version = -1;
     }

addons/addon-webgl/src/RectangleRenderer.ts

@@ -336,8 +336,9 @@ export class RectangleRenderer extends Disposable {
       }
     }
 
-    if (vertices.attributes.length < offset + 4) {
+    if (vertices.attributes.length < offset + INDICES_PER_RECTANGLE) {
-      vertices.attributes = expandFloat32Array(vertices.attributes, this._terminal.rows * this._terminal.cols * INDICES_PER_RECTANGLE);
+      // +1 for the viewport-clear rectangle at offset 0.
+      vertices.attributes = expandFloat32Array(vertices.attributes, (this._terminal.rows * this._terminal.cols + 1) * INDICES_PER_RECTANGLE);
     }
     $x1 = startX * this._dimensions.device.cell.width;
     $y1 = y * this._dimensions.device.cell.height;

addons/addon-webgl/src/TextureAtlas.ts

@@ -133,7 +133,9 @@ export class TextureAtlas implements ITextureAtlas {
 
   private _requestClearModel = false;
   public beginFrame(): boolean {
-    return this._requestClearModel;
+    const result = this._requestClearModel;
+    this._requestClearModel = false;
+    return result;
   }
 
   public clearTexture(): void {
@@ -193,7 +195,7 @@ export class TextureAtlas implements ITextureAtlas {
 
       // Merge into the new page
       const mergedPage = this._mergePages(mergingPages, mergedPageIndex);
-      mergedPage.version++;
+      mergedPage.version = ++AtlasPage.nextVersion;
 
       // Delete the pages, shifting glyph texture pages as needed
       for (let i = sortedMergingPagesIndexes.length - 1; i >= 0; i--) {
@@ -251,7 +253,7 @@ export class TextureAtlas implements ITextureAtlas {
       for (const g of adjustingPage.glyphs) {
         g.texturePage--;
       }
-      adjustingPage.version++;
+      adjustingPage.version = ++AtlasPage.nextVersion;
     }
   }
 
@@ -937,7 +939,7 @@ export class TextureAtlas implements ITextureAtlas {
       rasterizedGlyph.size.y
     );
     activePage.addGlyph(rasterizedGlyph);
-    activePage.version++;
+    activePage.version = ++AtlasPage.nextVersion;
 
     return rasterizedGlyph;
   }
@@ -1047,9 +1049,13 @@ class AtlasPage {
   }
 
   /**
-   * Used to check whether the canvas of the atlas page has changed.
+   * Monotonically increasing across all atlas pages globally. Used to detect when the texture
+   * unit at a given index needs to be re-uploaded — both for content changes within the same
+   * page and for a page object swap at the same index (which happens after a page merge,
+   * where a per-page counter could coincide with the previously-bound page's value).
    */
-  public version = 0;
+  public static nextVersion: number = 0;
+  public version = ++AtlasPage.nextVersion;
 
   // Texture atlas current positioning data. The texture packing strategy used is to fill from
   // left-to-right and top-to-bottom. When the glyph being written is less than half of the current
@@ -1092,7 +1098,7 @@ class AtlasPage {
     this.currentRow.y = 0;
     this.currentRow.height = 0;
     this.fixedRows.length = 0;
-    this.version++;
+    this.version = ++AtlasPage.nextVersion;
   }
 }
 

addons/addon-webgl/src/WebglRenderer.ts

@@ -28,6 +28,10 @@ import { addDisposableListener } from 'browser/Dom';
 import { combinedDisposable, Disposable, MutableDisposable, toDisposable } from 'common/Lifecycle';
 import { createRenderDimensions } from 'browser/renderer/shared/RendererUtils';
 
+const enum Constants {
+  MERGE_RETRY_LIMIT = 32
+}
+
 export class WebglRenderer extends Disposable implements IRenderer {
   private _renderLayers: IRenderLayer[];
   private _cursorBlinkStateManager: MutableDisposable<CursorBlinkStateManager> = this._register(new MutableDisposable());
@@ -375,6 +379,19 @@ export class WebglRenderer extends Disposable implements IRenderer {
       this._updateModel(start, end);
     }
 
+    // A mid-update atlas page merge invalidates vertex data and may not bump the host
+    // page's version, so re-run the update and force a full texture rebind.
+    let merged = false;
+    let mergeRetries = 0;
+    while (this._charAtlas && this._glyphRenderer.value.beginFrame() && mergeRetries++ < Constants.MERGE_RETRY_LIMIT) {
+      merged = true;
+      this._clearModel(true);
+      this._updateModel(0, this._terminal.rows - 1);
+    }
+    if (merged) {
+      this._glyphRenderer.value.invalidateAtlasTextures();
+    }
+
     // Render
     this._rectangleRenderer.value.renderBackgrounds();
     this._glyphRenderer.value.render(this._model);