Align browser registerMarker alt-buffer guard with headless

Block registerMarker on the alternate buffer in CoreBrowserTerminal,
matching headless addMarker behavior. Update public typings to
IMarker | undefined and add regression tests.

Fixes #5937

Co-authored-by: Daniel Imms <Tyriar@users.noreply.github.com>

src/browser/CoreBrowserTerminal.ts

@@ -768,7 +768,12 @@ export class CoreBrowserTerminal extends CoreTerminal implements ITerminal {
     return this.buffer.markers;
   }
 
-  public registerMarker(cursorYOffset: number): IMarker {
+  public registerMarker(cursorYOffset: number): IMarker | undefined {
+    // Disallow markers on the alt buffer
+    if (this.buffer !== this.buffers.normal) {
+      return;
+    }
+
     return this.buffer.addMarker(this.buffer.ybase + this.buffer.y + cursorYOffset);
   }
 

src/browser/Terminal.test.ts

@@ -1146,6 +1146,16 @@ describe('Terminal', () => {
     assert.equal(termConverting.buffer.lines.get(1)!.translateToString(true), 'World');
   });
 
+  describe('registerMarker', () => {
+    it('should return undefined on the alt buffer', async () => {
+      assert.isDefined(term.registerMarker(0));
+      await term.writeP('\x1b[?47h');
+      assert.isUndefined(term.registerMarker(0));
+      await term.writeP('\x1b[?47l');
+      assert.isDefined(term.registerMarker(0));
+    });
+  });
+
   // FIXME: move to common/CoreTerminal.test once the trimming is moved over
   describe('marker lifecycle', () => {
     // create a 10x5 terminal with markers on every line

src/browser/public/Terminal.ts

@@ -166,7 +166,7 @@ export class Terminal extends Disposable implements ITerminalApi {
   public deregisterCharacterJoiner(joinerId: number): void {
     this._core.deregisterCharacterJoiner(joinerId);
   }
-  public registerMarker(cursorYOffset: number = 0): IMarker {
+  public registerMarker(cursorYOffset: number = 0): IMarker | undefined {
     this._verifyIntegers(cursorYOffset);
     return this._core.registerMarker(cursorYOffset);
   }

src/headless/public/Terminal.test.ts

@@ -419,6 +419,18 @@ describe('Headless API Tests', function (): void {
       strictEqual(term.buffer.normal.getLine(0)!.translateToString(), 'norm ');
       strictEqual(term.buffer.alternate.getLine(0), undefined);
     });
+
+    it('registerMarker returns undefined on the alt buffer', async () => {
+      term = new Terminal({ cols: 5, allowProposedApi: true });
+      strictEqual(term.buffer.active.type, 'normal');
+      strictEqual(term.registerMarker(0)!.line, 0);
+      await writeSync('\x1b[?47h');
+      strictEqual(term.buffer.active.type, 'alternate');
+      strictEqual(term.registerMarker(0), undefined);
+      await writeSync('\x1b[?47l');
+      strictEqual(term.buffer.active.type, 'normal');
+      strictEqual(term.registerMarker(0)!.line, 0);
+    });
   });
 
   describe('modes', () => {