Allow selection in the a11y tree and sync the selection to terminal

css/xterm.css

@@ -140,7 +140,7 @@
     cursor: crosshair;
 }
 
-.xterm .xterm-accessibility,
+.xterm .xterm-accessibility:not(.debug),
 .xterm .xterm-message {
     position: absolute;
     left: 0;
@@ -152,6 +152,15 @@
     pointer-events: none;
 }
 
+.xterm .xterm-accessibility-tree:not(.debug) *::selection {
+  color: transparent;
+}
+
+.xterm .xterm-accessibility-tree {
+  user-select: text;
+  white-space: pre;
+}
+
 .xterm .live-region {
     position: absolute;
     left: -9999px;

src/browser/AccessibilityManager.ts

@@ -10,6 +10,7 @@ import { Disposable, toDisposable } from 'common/Lifecycle';
 import { ICoreBrowserService, IRenderService } from 'browser/services/Services';
 import { IBuffer } from 'common/buffer/Types';
 import { IInstantiationService } from 'common/services/Services';
+import { addDisposableDomListener } from 'browser/Lifecycle';
 
 const MAX_ROWS_TO_READ = 20;
 
@@ -18,11 +19,17 @@ const enum BoundaryPosition {
   BOTTOM
 }
 
+// Turn this on to unhide the accessibility tree and display it under
+// (instead of overlapping with) the terminal.
+const DEBUG = false;
+
 export class AccessibilityManager extends Disposable {
+  private _debugRootContainer: HTMLElement | undefined;
   private _accessibilityContainer: HTMLElement;
 
   private _rowContainer: HTMLElement;
   private _rowElements: HTMLElement[];
+  private _rowColumns: WeakMap<HTMLElement, number[]> = new WeakMap();
 
   private _liveRegion: HTMLElement;
   private _liveRegionLineCount: number = 0;
@@ -80,7 +87,23 @@ export class AccessibilityManager extends Disposable {
     if (!this._terminal.element) {
       throw new Error('Cannot enable accessibility before Terminal.open');
     }
-    this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
+
+    if (DEBUG) {
+      this._accessibilityContainer.classList.add('debug');
+      this._rowContainer.classList.add('debug');
+
+      // Use a `<div class="xterm">` container so that the css will still apply.
+      this._debugRootContainer = document.createElement('div');
+      this._debugRootContainer.classList.add('xterm');
+
+      this._debugRootContainer.appendChild(document.createTextNode('------start a11y------'));
+      this._debugRootContainer.appendChild(this._accessibilityContainer);
+      this._debugRootContainer.appendChild(document.createTextNode('------end a11y------'));
+
+      this._terminal.element.insertAdjacentElement('afterend', this._debugRootContainer);
+    } else {
+      this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
+    }
 
     this.register(this._terminal.onResize(e => this._handleResize(e.rows)));
     this.register(this._terminal.onRender(e => this._refreshRows(e.start, e.end)));
@@ -92,11 +115,16 @@ export class AccessibilityManager extends Disposable {
     this.register(this._terminal.onKey(e => this._handleKey(e.key)));
     this.register(this._terminal.onBlur(() => this._clearLiveRegion()));
     this.register(this._renderService.onDimensionsChange(() => this._refreshRowsDimensions()));
+    this.register(addDisposableDomListener(document, 'selectionchange', () => this._handleSelectionChange()));
     this.register(this._coreBrowserService.onDprChange(() => this._refreshRowsDimensions()));
 
     this._refreshRows();
     this.register(toDisposable(() => {
-      this._accessibilityContainer.remove();
+      if (DEBUG) {
+        this._debugRootContainer!.remove();
+      } else {
+        this._accessibilityContainer.remove();
+      }
       this._rowElements.length = 0;
     }));
   }
@@ -149,14 +177,18 @@ export class AccessibilityManager extends Disposable {
     const buffer: IBuffer = this._terminal.buffer;
     const setSize = buffer.lines.length.toString();
     for (let i = start; i <= end; i++) {
-      const lineData = buffer.translateBufferLineToString(buffer.ydisp + i, true);
+      const line = buffer.lines.get(buffer.ydisp + i);
+      const columns: number[] = [];
+      const lineData = line?.translateToString(true, undefined, undefined, columns) || '';
       const posInSet = (buffer.ydisp + i + 1).toString();
       const element = this._rowElements[i];
       if (element) {
         if (lineData.length === 0) {
           element.innerText = '\u00a0';
+          this._rowColumns.set(element, [0, 1]);
         } else {
           element.textContent = lineData;
+          this._rowColumns.set(element, columns);
         }
         element.setAttribute('aria-posinset', posInSet);
         element.setAttribute('aria-setsize', setSize);
@@ -233,6 +265,103 @@ export class AccessibilityManager extends Disposable {
     e.stopImmediatePropagation();
   }
 
+  private _handleSelectionChange(): void {
+    if (this._rowElements.length === 0) {
+      return;
+    }
+
+    const selection = document.getSelection();
+    if (!selection) {
+      return;
+    }
+
+    if (selection.isCollapsed) {
+      // Only do something when the anchorNode is inside the row container. This
+      // behavior mirrors what we do with mouse --- if the mouse clicks
+      // somewhere outside of the terminal, we don't clear the selection.
+      if (this._rowContainer.contains(selection.anchorNode)) {
+        this._terminal.clearSelection();
+      }
+      return;
+    }
+
+    if (!selection.anchorNode || !selection.focusNode) {
+      console.error('anchorNode and/or focusNode are null');
+      return;
+    }
+
+    // Sort the two selection points in document order.
+    let begin = { node: selection.anchorNode, offset: selection.anchorOffset };
+    let end = { node: selection.focusNode, offset: selection.focusOffset };
+    if ((begin.node.compareDocumentPosition(end.node) & Node.DOCUMENT_POSITION_PRECEDING) || (begin.node === end.node && begin.offset > end.offset) ) {
+      [begin, end] = [end, begin];
+    }
+
+    // Clamp begin/end to the inside of the row container.
+    if (begin.node.compareDocumentPosition(this._rowElements[0]) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_FOLLOWING)) {
+      begin = { node: this._rowElements[0].childNodes[0], offset: 0 };
+    }
+    if (!this._rowContainer.contains(begin.node)) {
+      // This happens when `begin` is below the last row.
+      return;
+    }
+    const lastRowElement = this._rowElements.slice(-1)[0];
+    if (end.node.compareDocumentPosition(lastRowElement) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_PRECEDING)) {
+      end = {
+        node: lastRowElement,
+        offset: lastRowElement.textContent?.length ?? 0
+      };
+    }
+    if (!this._rowContainer.contains(end.node)) {
+      // This happens when `end` is above the first row.
+      return;
+    }
+
+    const toRowColumn = ({ node, offset }: typeof begin): {row: number, column: number} | null => {
+      // `node` is either the row element or the Text node inside it.
+      const rowElement: any = node instanceof Text ? node.parentNode : node;
+      let row = parseInt(rowElement?.getAttribute('aria-posinset'), 10) - 1;
+      if (isNaN(row)) {
+        console.warn('row is invalid. Race condition?');
+        return null;
+      }
+
+      const columns = this._rowColumns.get(rowElement);
+      if (!columns) {
+        console.warn('columns is null. Race condition?');
+        return null;
+      }
+
+      let column = offset < columns.length ? columns[offset] : columns.slice(-1)[0] + 1;
+      if (column >= this._terminal.cols) {
+        ++row;
+        column = 0;
+      }
+      return {
+        row,
+        column
+      };
+    };
+
+    const beginRowColumn = toRowColumn(begin);
+    const endRowColumn = toRowColumn(end);
+
+    if (!beginRowColumn || !endRowColumn) {
+      return;
+    }
+
+    if (beginRowColumn.row > endRowColumn.row || (beginRowColumn.row === endRowColumn.row && beginRowColumn.column >= endRowColumn.column)) {
+      // This should not happen unless we have some bugs.
+      throw new Error('invalid range');
+    }
+
+    this._terminal.select(
+      beginRowColumn.column,
+      beginRowColumn.row,
+      (endRowColumn.row - beginRowColumn.row) * this._terminal.cols - beginRowColumn.column + endRowColumn.column
+    );
+  }
+
   private _handleResize(rows: number): void {
     // Remove bottom boundary listener
     this._rowElements[this._rowElements.length - 1].removeEventListener('focus', this._bottomBoundaryFocusListener);

src/common/Types.d.ts

@@ -247,7 +247,7 @@ export interface IBufferLine {
   clone(): IBufferLine;
   getTrimmedLength(): number;
   getNoBgTrimmedLength(): number;
-  translateToString(trimRight?: boolean, startCol?: number, endCol?: number): string;
+  translateToString(trimRight?: boolean, startCol?: number, endCol?: number, outColumns?: number[]): string;
 
   /* direct access to cell attrs */
   getWidth(index: number): number;