Skip to content

Commit

Permalink
Allow selection in the a11y tree and sync the selection to terminal
Browse files Browse the repository at this point in the history
  • Loading branch information
Jason Lin committed Sep 1, 2023
1 parent 2dd4dfd commit 7800989
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 52 deletions.
9 changes: 9 additions & 0 deletions css/xterm.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,15 @@
pointer-events: none;
}

.xterm .xterm-accessibility-tree *::selection {
color: transparent;
}

.xterm .xterm-accessibility-tree {
user-select: text;
white-space: pre;
}

.xterm .live-region {
position: absolute;
left: -9999px;
Expand Down
122 changes: 119 additions & 3 deletions src/browser/AccessibilityManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export class AccessibilityManager extends Disposable {

private _rowContainer: HTMLElement;
private _rowElements: HTMLElement[];
private _rowColumns: WeakMap<HTMLElement, number[]> = new WeakMap();

private _liveRegion: HTMLElement;
private _liveRegionLineCount: number = 0;
Expand Down Expand Up @@ -53,8 +54,15 @@ export class AccessibilityManager extends Disposable {
@IRenderService private readonly _renderService: IRenderService
) {
super();

// Turn this on to unhide the accessibility tree and display it under
// (instead of overlapping with) the terminal.
const debug = false;

this._accessibilityContainer = document.createElement('div');
this._accessibilityContainer.classList.add('xterm-accessibility');
if (!debug) {
this._accessibilityContainer.classList.add('xterm-accessibility');
}

this._rowContainer = document.createElement('div');
this._rowContainer.setAttribute('role', 'list');
Expand Down Expand Up @@ -82,7 +90,13 @@ export class AccessibilityManager extends Disposable {
if (!this._terminal.element) {
throw new Error('Cannot enable accessibility before Terminal.open');
}
this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
if (!debug) {
this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
} else {
this._terminal.element.insertAdjacentElement('afterend', this._accessibilityContainer);
this._accessibilityContainer.insertAdjacentText('beforebegin', '-----start-----');
this._accessibilityContainer.insertAdjacentText('afterend', '------end------');
}

this.register(this._terminal.onResize(e => this._handleResize(e.rows)));
this.register(this._terminal.onRender(e => this._refreshRows(e.start, e.end)));
Expand All @@ -94,6 +108,7 @@ export class AccessibilityManager extends Disposable {
this.register(this._terminal.onKey(e => this._handleKey(e.key)));
this.register(this._terminal.onBlur(() => this._clearLiveRegion()));
this.register(this._renderService.onDimensionsChange(() => this._refreshRowsDimensions()));
this.register(addDisposableDomListener(document, 'selectionchange', () => this._handleSelectionChange()));

this._screenDprMonitor = new ScreenDprMonitor(window);
this.register(this._screenDprMonitor);
Expand Down Expand Up @@ -171,14 +186,18 @@ export class AccessibilityManager extends Disposable {
const buffer: IBuffer = this._terminal.buffer;
const setSize = buffer.lines.length.toString();
for (let i = start; i <= end; i++) {
const lineData = buffer.translateBufferLineToString(buffer.ydisp + i, true);
const line = buffer.lines.get(buffer.ydisp + i);
const columns: number[] = [];
const lineData = line?.translateToString(true, undefined, undefined, columns) || '';
const posInSet = (buffer.ydisp + i + 1).toString();
const element = this._rowElements[i];
if (element) {
if (lineData.length === 0) {
element.innerText = '\u00a0';
this._rowColumns.set(element, [0]);
} else {
element.textContent = lineData;
this._rowColumns.set(element, columns);
}
element.setAttribute('aria-posinset', posInSet);
element.setAttribute('aria-setsize', setSize);
Expand Down Expand Up @@ -255,6 +274,103 @@ export class AccessibilityManager extends Disposable {
e.stopImmediatePropagation();
}

private _handleSelectionChange(): void {
if (this._rowElements.length === 0) {
return;
}

const selection = document.getSelection();
if (!selection) {
return;
}

if (selection.isCollapsed) {
// Only do something when the anchorNode is inside the row container. This
// behavior mirrors what we do with mouse --- if the mouse clicks
// somewhere outside of the terminal, we don't clear the selection.
if (this._rowContainer.contains(selection.anchorNode)) {
this._terminal.clearSelection();
}
return;
}

if (!selection.anchorNode || !selection.focusNode) {
console.error('anchorNode and/or focusNode are null');
return;
}

// Sort the two selection points in document order.
let begin = { node: selection.anchorNode, offset: selection.anchorOffset };
let end = { node: selection.focusNode, offset: selection.focusOffset };
if ((begin.node.compareDocumentPosition(end.node) & Node.DOCUMENT_POSITION_PRECEDING) || (begin.node === end.node && begin.offset > end.offset) ) {
[begin, end] = [end, begin];
}

// Clamp begin/end to the inside of the row container.
if (begin.node.compareDocumentPosition(this._rowElements[0]) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_FOLLOWING)) {
begin = { node: this._rowElements[0].childNodes[0], offset: 0 };
}
if (!this._rowContainer.contains(begin.node)) {
// This happens when `begin` is below the last row.
return;
}
const lastRowElement = this._rowElements.slice(-1)[0];
if (end.node.compareDocumentPosition(lastRowElement) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_PRECEDING)) {
end = {
node: lastRowElement,
offset: lastRowElement.textContent?.length ?? 0
};
}
if (!this._rowContainer.contains(end.node)) {
// This happens when `end` is above the first row.
return;
}

const toRowColumn = ({ node, offset }: typeof begin): {row: number, column: number} | null => {
// `node` is either the row element or the Text node inside it.
const rowElement: any = node instanceof Text ? node.parentNode : node;
let row = parseInt(rowElement?.getAttribute('aria-posinset'), 10) - 1;
if (isNaN(row)) {
console.warn('row is invalid. Race condition?');
return null;
}

const columns = this._rowColumns.get(rowElement);
if (!columns) {
console.warn('columns is null. Race condition?');
return null;
}

let column = offset < columns.length ? columns[offset] : columns.slice(-1)[0] + 1;
if (column >= this._terminal.cols) {
++row;
column = 0;
}
return {
row,
column
};
};

const beginRowColumn = toRowColumn(begin);
const endRowColumn = toRowColumn(end);

if (!beginRowColumn || !endRowColumn) {
return;
}

if (beginRowColumn.row > endRowColumn.row || (beginRowColumn.row === endRowColumn.row && beginRowColumn.column >= endRowColumn.column)) {
// This should not happen unless we have some bugs.
throw new Error('invalid range');
}

this._terminal.select(
beginRowColumn.column,
beginRowColumn.row,
(endRowColumn.row - beginRowColumn.row) * this._terminal.cols - beginRowColumn.column + endRowColumn.column
);
}

private _handleResize(rows: number): void {
// Remove bottom boundary listener
this._rowElements[this._rowElements.length - 1].removeEventListener('focus', this._bottomBoundaryFocusListener);
Expand Down
2 changes: 1 addition & 1 deletion src/common/Types.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ export interface IBufferLine {
clone(): IBufferLine;
getTrimmedLength(): number;
getNoBgTrimmedLength(): number;
translateToString(trimRight?: boolean, startCol?: number, endCol?: number): string;
translateToString(trimRight?: boolean, startCol?: number, endCol?: number, outColumns?: number[]): string;

/* direct access to cell attrs */
getWidth(index: number): number;
Expand Down
121 changes: 76 additions & 45 deletions src/common/buffer/BufferLine.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,56 +331,75 @@ describe('BufferLine', function(): void {
describe('translateToString with and w\'o trimming', function(): void {
it('empty line', function(): void {
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
assert.equal(line.translateToString(false), ' ');
assert.equal(line.translateToString(true), '');
const columns: number[] = [];
assert.equal(line.translateToString(false, undefined, undefined, columns), ' ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), '');
assert.deepEqual(columns, []);
});
it('ASCII', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a a aa ');
assert.equal(line.translateToString(true), 'a a aa');
assert.equal(line.translateToString(false, 0, 5), 'a a a');
assert.equal(line.translateToString(false, 0, 4), 'a a ');
assert.equal(line.translateToString(false, 0, 3), 'a a');
assert.equal(line.translateToString(true, 0, 5), 'a a a');
assert.equal(line.translateToString(true, 0, 4), 'a a ');
assert.equal(line.translateToString(true, 0, 3), 'a a');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a a aa');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a a a');
assert.deepEqual(columns, [0, 1, 2, 3, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a a ');
assert.deepEqual(columns, [0, 1, 2, 3]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a a');
assert.deepEqual(columns, [0, 1, 2]);
}

});
it('surrogate', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a 𝄞 𝄞𝄞 ');
assert.equal(line.translateToString(true), 'a 𝄞 𝄞𝄞');
assert.equal(line.translateToString(false, 0, 5), 'a 𝄞 𝄞');
assert.equal(line.translateToString(false, 0, 4), 'a 𝄞 ');
assert.equal(line.translateToString(false, 0, 3), 'a 𝄞');
assert.equal(line.translateToString(true, 0, 5), 'a 𝄞 𝄞');
assert.equal(line.translateToString(true, 0, 4), 'a 𝄞 ');
assert.equal(line.translateToString(true, 0, 3), 'a 𝄞');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a 𝄞 𝄞𝄞 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a 𝄞 𝄞𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a 𝄞 𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a 𝄞 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a 𝄞');
assert.deepEqual(columns, [0, 1, 2, 2]);
}
});
it('combining', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a e\u0301 e\u0301e\u0301 ');
assert.equal(line.translateToString(true), 'a e\u0301 e\u0301e\u0301');
assert.equal(line.translateToString(false, 0, 5), 'a e\u0301 e\u0301');
assert.equal(line.translateToString(false, 0, 4), 'a e\u0301 ');
assert.equal(line.translateToString(false, 0, 3), 'a e\u0301');
assert.equal(line.translateToString(true, 0, 5), 'a e\u0301 e\u0301');
assert.equal(line.translateToString(true, 0, 4), 'a e\u0301 ');
assert.equal(line.translateToString(true, 0, 3), 'a e\u0301');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a e\u0301 e\u0301e\u0301 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a e\u0301 e\u0301e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a e\u0301 e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a e\u0301 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2]);
}
});
it('fullwidth', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, '１', 2, '１'.charCodeAt(0)]));
Expand All @@ -389,43 +408,55 @@ describe('BufferLine', function(): void {
line.setCell(6, CellData.fromCharData([0, '', 0, 0]));
line.setCell(7, CellData.fromCharData([1, '１', 2, '１'.charCodeAt(0)]));
line.setCell(8, CellData.fromCharData([0, '', 0, 0]));
assert.equal(line.translateToString(false), 'a １ １１ ');
assert.equal(line.translateToString(true), 'a １ １１');
assert.equal(line.translateToString(false, 0, 7), 'a １ １');
assert.equal(line.translateToString(false, 0, 6), 'a １ １');
assert.equal(line.translateToString(false, 0, 5), 'a １ ');
assert.equal(line.translateToString(false, 0, 4), 'a １');
assert.equal(line.translateToString(false, 0, 3), 'a １');
assert.equal(line.translateToString(false, 0, 2), 'a ');
assert.equal(line.translateToString(true, 0, 7), 'a １ １');
assert.equal(line.translateToString(true, 0, 6), 'a １ １');
assert.equal(line.translateToString(true, 0, 5), 'a １ ');
assert.equal(line.translateToString(true, 0, 4), 'a １');
assert.equal(line.translateToString(true, 0, 3), 'a １');
assert.equal(line.translateToString(true, 0, 2), 'a ');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a １ １１ ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a １ １１');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 7, columns), 'a １ １');
assert.deepEqual(columns, [0, 1, 2, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 6, columns), 'a １ １');
assert.deepEqual(columns, [0, 1, 2, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a １ ');
assert.deepEqual(columns, [0, 1, 2, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a １');
assert.deepEqual(columns, [0, 1, 2]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a １');
assert.deepEqual(columns, [0, 1, 2]);
assert.equal(line.translateToString(trimRight, 0, 2, columns), 'a ');
assert.deepEqual(columns, [0, 1]);
}
});
it('space at end', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(6, CellData.fromCharData([1, ' ', 1, ' '.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a a aa ');
assert.equal(line.translateToString(true), 'a a aa ');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6]);
});
it('should always return some sane value', function(): void {
const columns: number[] = [];
// sanity check - broken line with invalid out of bound null width cells
// this can atm happen with deleting/inserting chars in inputhandler by "breaking"
// fullwidth pairs --> needs to be fixed after settling BufferLine impl
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, 0, NULL_CELL_CODE]), false);
assert.equal(line.translateToString(false), ' ');
assert.equal(line.translateToString(true), '');
assert.equal(line.translateToString(false, undefined, undefined, columns), ' ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), '');
assert.deepEqual(columns, []);
});
it('should work with endCol=0', () => {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, 0, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
assert.equal(line.translateToString(true, 0, 0), '');
assert.equal(line.translateToString(true, 0, 0, columns), '');
assert.deepEqual(columns, []);
});
});
describe('addCharToCell', () => {
Expand Down
Loading

0 comments on commit 7800989

Please sign in to comment.