From fb591fbe17503b10d64f74e847dc7c07fbcdeb99 Mon Sep 17 00:00:00 2001 From: Ben Schmidt Date: Mon, 9 Sep 2024 00:21:36 -0400 Subject: [PATCH] use a shared object with multiple length keys to store regl buffers --- src/regl_rendering.ts | 364 ++++++++++++++++++++---------------------- src/scatterplot.ts | 53 ++---- src/tile.ts | 6 +- src/types.ts | 11 +- 4 files changed, 194 insertions(+), 240 deletions(-) diff --git a/src/regl_rendering.ts b/src/regl_rendering.ts index 4a44c67d9..10f75f8c9 100644 --- a/src/regl_rendering.ts +++ b/src/regl_rendering.ts @@ -30,13 +30,9 @@ import { } from './aesthetics/StatefulAesthetic'; import { Scatterplot } from './scatterplot'; import { - Bool, Data, Dictionary, - Float, - Int, StructRowProxy, - Timestamp, Type, Utf8, Vector, @@ -45,6 +41,8 @@ import { Color } from './aesthetics/ColorAesthetic'; import { StatefulAesthetic } from './aesthetics/StatefulAesthetic'; import { Filter, Foreground } from './aesthetics/BooleanAesthetic'; import { ZoomTransform } from 'd3-zoom'; +import { TupleMap } from './utilityFunctions'; +import { buffer } from 'd3'; // eslint-disable-next-line import/prefer-default-export export class ReglRenderer extends Renderer { public regl: Regl; @@ -67,7 +65,7 @@ export class ReglRenderer extends Renderer { public contour_alpha_vals?: Float32Array | Uint8Array | Uint16Array; public tick_num?: number; public reglframe?: REGL.Cancellable; - public _integer_buffer?: Buffer; + public bufferManager: BufferManager; // public _renderer : Renderer; constructor( @@ -76,6 +74,7 @@ export class ReglRenderer extends Renderer { scatterplot: Scatterplot, ) { super(selector, scatterplot); + const c = this.canvas; if (this.canvas === undefined) { throw new Error('No canvas found'); @@ -91,6 +90,7 @@ export class ReglRenderer extends Renderer { ], canvas: c, }); + this.deeptable = tileSet; this.aes = new AestheticSet(scatterplot, this.regl, tileSet); @@ -108,6 +108,7 @@ export class ReglRenderer extends Renderer { })), void this.initialize(); this._buffers = new MultipurposeBufferSet(this.regl, this.buffer_size); + this.bufferManager = new BufferManager(this); } get buffers() { @@ -224,17 +225,21 @@ export class ReglRenderer extends Renderer { props.background_draw_needed[0] || props.background_draw_needed[1]; for (const tile of this.visible_tiles()) { // Do the binding operation; returns truthy if it's already done. - tile._buffer_manager = - tile._buffer_manager || new TileBufferManager(this.regl, tile, this); - - if (!tile._buffer_manager.ready()) { + if ( + !this.bufferManager.ready( + tile, + this.needeedFields.map((d) => [d]), + ) + ) { continue; } + const this_props = { - manager: tile._buffer_manager, number: call_no++, foreground_draw_number: needs_background_pass ? 1 : 1, tile_id: tile.numeric_id, + tile: tile, + count: tile.manifest.nPoints, ...props, } as DS.TileDrawProps; prop_list.push(this_props); @@ -251,8 +256,8 @@ export class ReglRenderer extends Renderer { return ( (3 + a.foreground_draw_number) * 1000 - (3 + b.foreground_draw_number) * 1000 + - b.number - - a.number + b.tile_id - + a.tile_id ); }); this._renderer(prop_list); @@ -292,8 +297,6 @@ export class ReglRenderer extends Renderer { ); } - const start = Date.now(); - try { this.render_all(props); } catch (error) { @@ -439,67 +442,6 @@ export class ReglRenderer extends Renderer { } } - /* - set_image_data(tile, ix) { - // Stores a *single* image onto the texture. - const { regl } = this; - - this.initialize_sprites(tile); - - // const { sprites, image_locations } = tile._regl_elements; - const { current_position } = sprites; - if (current_position[1] > (4096 - 18 * 2)) { - console.error(`First spritesheet overflow on ${tile.key}`); - // Just move back to the beginning. Will cause all sorts of havoc. - sprites.current_position = [0, 0]; - return; - } - if (!tile.table.get(ix)._jpeg) { - - } - } - */ - /* - spritesheet_setter(word) { - // Set if not there. - let ctx = 0; - if (!this.spritesheet) { - const offscreen = create('canvas') - .attr('width', 4096) - .attr('width', 4096) - .style('display', 'none'); - - ctx = offscreen.node().getContext('2d'); - const font_size = 32; - ctx.font = `${font_size}px Times New Roman`; - ctx.fillStyle = 'black'; - ctx.lookups = new Map(); - ctx.position = [0, font_size - font_size / 4.0]; - this.spritesheet = ctx; - } else { - ctx = this.spritesheet; - } - let [x, y] = ctx.position; - - if (ctx.lookups.get(word)) { - return ctx.lookups.get(word); - } - const w_ = ctx.measureText(word).width; - if (w_ > 4096) { - return; - } - if ((x + w_) > 4096) { - x = 0; - y += font_size; - } - ctx.fillText(word, x, y); - lookups.set(word, { x, y, width: w_ }); - // ctx.strokeRect(x, y - font_size, width, font_size) - x += w_; - ctx.position = [x, y]; - return lookups.get(word); - } - */ initialize_textures() { const { regl } = this; this.fbos = this.fbos || {}; @@ -603,17 +545,6 @@ export class ReglRenderer extends Renderer { return v; } - get integer_buffer(): wrapREGL.Buffer { - if (this._integer_buffer === undefined) { - const array = new Float32Array(2 ** 16); - for (let i = 0; i < 2 ** 16; i++) { - array[i] = i; - } - this._integer_buffer = this.regl.buffer(array); - } - return this._integer_buffer; - } - color_pick(x: number, y: number): null | StructRowProxy { if (y === 0) { // Not sure why, but this makes things complainy. @@ -755,7 +686,7 @@ export class ReglRenderer extends Renderer { frag: frag_shader as string, vert: vertex_shader as string, count(_, props) { - return props.manager.count; + return props.count; }, attributes: {}, uniforms: { @@ -875,9 +806,12 @@ export class ReglRenderer extends Renderer { for (const i of range(0, 16)) { parameters.attributes[`buffer_${i}`] = ( _, - { manager, buffer_num_to_variable }: P, + { tile, buffer_num_to_variable }: P, ) => { - const c = manager.regl_elements.get(buffer_num_to_variable[i]); + if (i >= buffer_num_to_variable.length) { + return { constant: 0 }; + } + const c = this.bufferManager.get([tile, ...buffer_num_to_variable[i]]); return c || { constant: 0 }; }; } @@ -1026,7 +960,9 @@ export class ReglRenderer extends Renderer { } } - const buffer_num_to_variable = [...Object.keys(variable_to_buffer_num)]; + const buffer_num_to_variable = [ + ...Object.keys(variable_to_buffer_num).map((k) => [k]), + ]; this.aes_to_buffer_num = aes_to_buffer_num; this.variable_to_buffer_num = variable_to_buffer_num; this.buffer_num_to_variable = buffer_num_to_variable; @@ -1034,7 +970,7 @@ export class ReglRenderer extends Renderer { aes_to_buffer_num?: Record; variable_to_buffer_num?: Record; - buffer_num_to_variable?: string[]; + buffer_num_to_variable?: string[][]; get discard_share() { // If jitter is temporal, e.g., or filters are in place, @@ -1043,8 +979,8 @@ export class ReglRenderer extends Renderer { } } -export class TileBufferManager { - // Handle the interactions of a tile with a regl state. +export class BufferManager { + // Handle the interactions of tiles with a regl state. // binds elements directly to the tile, so it's safe // to re-run this multiple times on the same tile. @@ -1053,28 +989,50 @@ export class TileBufferManager { // but since they relate to Regl, // I want them in this file instead. - public tile: Tile; - public regl: Regl; + private regl: Regl; public renderer: ReglRenderer; - public regl_elements: Map; - - constructor(regl: Regl, tile: Tile, renderer: ReglRenderer) { - this.tile = tile; - this.regl = regl; + private bufferMap: WeakMap = new Map(); + private arrayMap: TupleMap = new TupleMap(); + public ixInTileBuffer: DS.BufferLocation; + // A list of integers. Used internally. + private _integer_array?: Float32Array; + // A buffer populated from that list. + private _integer_buffer?: Buffer; + + constructor(renderer: ReglRenderer) { + this.regl = renderer.regl; this.renderer = renderer; // Reuse the same buffer for all `ix_in_tile` keys, because // it's just a set of integers going up. - this.regl_elements = new Map([ - [ - 'ix_in_tile', - { - offset: 0, - stride: 4, - buffer: renderer.integer_buffer, - byte_size: 4 * 2 ** 16, - }, - ], - ]); + this.ixInTileBuffer = { + offset: 0, + stride: 4, + buffer: this.integer_buffer, + byte_size: 4 * 2 ** 16, + }; + } + + get integer_array(): Float32Array { + if (this._integer_array === undefined) { + this._integer_array = new Float32Array(2 ** 16); + for (let i = 0; i < 2 ** 16; i++) { + this._integer_array[i] = i; + } + } + return this._integer_array; + } + + get integer_buffer(): wrapREGL.Buffer { + if (this._integer_buffer === undefined) { + const array = this.integer_array; + this._integer_buffer = this.regl.buffer(array); + } + return this._integer_buffer; + } + + get(k: (string | Tile)[]): DS.BufferLocation | null { + const a = this.bufferMap.get(this.arrayMap.get(k)); + return a; } /** @@ -1082,99 +1040,67 @@ export class TileBufferManager { * @param * @returns Is the buffer ready with all the requested aesthetics for the current plot? */ - ready() { - const { renderer } = this; - + ready(tile: Tile, needed_dimensions: Iterable): boolean { // We don't allocate buffers for dimensions until they're needed. - // This code checks what buffers the current plot call is expecting. - const needed_dimensions: Set = new Set(); - for (const v of Object.values(renderer.aes.store)) { - for (const aesthetic of v.states) { - if (aesthetic.field) { - needed_dimensions.add(aesthetic.field); - } - } - } - for (const key of ['ix', 'ix_in_tile', ...needed_dimensions]) { - const current = this.regl_elements.get(key); + + for (const key of [['ix'], ['ix_in_tile'], ...needed_dimensions]) { + const current = this.get([tile, ...key]); if (current === null || current === undefined) { - if (this.tile.hasLoadedColumn(key)) { - this.create_regl_buffer(key); + if (tile.hasLoadedColumn(key[0])) { + this.create_regl_buffer(tile, key); } else { - return false; + // console.log('not ready because of', key); + if (key[0] === 'ix_in_tile') { + this.create_regl_buffer(tile, key); + } else { + return false; + } } } } return true; } - async awaitReady() {} /** * * @param colname the name of the column to release * * @returns Nothing, not even if the column isn't currently defined. */ - release(colname: string): void { - const current = this.regl_elements.get(colname); + release(tile: Tile, colname: string[]): void { + const current = this.get([tile, ...colname]); if (current) { this.renderer.buffers.free_block(current); } } - get count() { - return this.tile.record_batch.numRows; - } - - create_buffer_data(key: string): Float32Array { - const { tile } = this; - type ColumnType = Vector | Float | Bool | Int | Timestamp>; - - if (!tile.hasLoadedColumn(key)) { - if (tile.deeptable.transformations[key] !== undefined) { - throw new Error( - 'Attempted to create buffer data on an unloaded transformation', - ); - } else { - let col_names = [ - ...tile.record_batch.schema.fields.map((d) => d.name), - ...Object.keys(tile.deeptable.transformations), - ]; - if (!key.startsWith('_')) { - // Don't warn internal columns unless the user is in internal-column land. - col_names = col_names.filter((d) => !d.startsWith('_')); - } - throw new Error( - `Requested ${key} but table only has columns ["${col_names.join( - '", "', - )}]"`, - ); - } - } - - const column = tile.record_batch.getChild(key) as ColumnType; - if (column.data.length !== 1) { - throw new Error( - `Column ${key} has ${column.data.length} buffers, not 1.`, - ); - } - - if (!column.type || !column.type.typeId) { - throw new Error(`Column ${key} has no type.`); - } + /** + * + * @param tile the tile to use + * @param key the (nested?) key in the dataset. This must be already populated + * when this function is run. + * @returns A Float32-converted version of the data in the column + * suitable to be dropped onto a webGL buffer. + */ + convertToGlFormat(column: Vector): Float32Array { // Anything that isn't a single-precision float must be coerced to one. if (!column.type || column.type.typeId !== Type.Float32) { - const buffer = new Float32Array(tile.record_batch.numRows); + // For numeric data, it's safe to simply return the data straight up. + if (column.data[0].values.constructor === Float64Array) { + return new Float32Array(column.data[0].values); + } + + const buffer = new Float32Array(column.length); const source_buffer = column.data[0]; if (column.type.typeId === Type.Dictionary) { - for (let i = 0; i < tile.record_batch.numRows; i++) { + for (let i = 0; i < column.length; i++) { buffer[i] = (source_buffer as Data>).values[i]; } } else if (column.type.typeId === Type.Bool) { // Booleans are unpacked using arrow fundamentals unless we see // a reason to do it directly with bit operations (such as the null checks) // being expensive. - for (let i = 0; i < tile.record_batch.numRows; i++) { + for (let i = 0; i < column.length; i++) { buffer[i] = column.get(i) ? 1 : 0; } } else if ( @@ -1205,42 +1131,94 @@ export class TileBufferManager { throw new Error(`Unknown time type ${timetype}`); } - for (let i = 0; i < tile.record_batch.numRows; i++) { + for (let i = 0; i < column.length; i++) { buffer[i] = Number(view64[i]) / divisor; } } else { - for (let i = 0; i < tile.record_batch.numRows; i++) { + for (let i = 0; i < column.length; i++) { buffer[i] = Number(source_buffer.values[i]); } } return buffer; } - // For numeric data, it's safe to simply return the data straight up. - if (column.data[0].values.constructor === Float64Array) { - return new Float32Array(column.data[0].values); - } return column.data[0].values as Float32Array; } - create_regl_buffer(key: string): void { - const { regl_elements, renderer } = this; - if (regl_elements.has(key)) { + create_regl_buffer(tile: Tile, keys: string[]): void { + const { renderer } = this; + const key = [tile, ...keys]; + if (this.arrayMap.has(key)) { + return; + } + // console.log({ keys }); + if (keys[0] === 'ix_in_tile') { + this.arrayMap.set(key, this.integer_array); + if (!this.bufferMap.has(this.integer_array)) { + this.bufferMap.set(this.integer_array, this.ixInTileBuffer); + } return; } - const data = this.create_buffer_data(key); - if (data.constructor !== Float32Array) { - console.warn(typeof data, data); - throw new Error('Buffer data must be a Float32Array'); + + const vector = getNestedVector(tile, keys); + this.arrayMap.set(key, vector.data[0].values); + + if (this.bufferMap.has(vector.data[0].values)) { + return; } + + const data = this.convertToGlFormat(vector); const item_size = 4; const data_length = data.length; const buffer_desc = renderer.buffers.allocate_block(data_length, item_size); + buffer_desc.buffer.subdata(data, buffer_desc.offset); - regl_elements.set(key, buffer_desc); + this.bufferMap.set(vector.data[0].values, buffer_desc); + } +} - buffer_desc.buffer.subdata(data, buffer_desc.offset); +// TODO: Build this out in next PR. +function getNestedVector( + tile: Tile, + key: string[], +): Vector { + if (!tile.hasLoadedColumn(key[0])) { + if (tile.deeptable.transformations[key[0]] !== undefined) { + throw new Error( + 'Attempted to create buffer data on an unloaded transformation', + ); + } else { + let col_names = [ + ...tile.record_batch.schema.fields.map((d) => d.name), + ...Object.keys(tile.deeptable.transformations), + ]; + if (!key[0].startsWith('_')) { + // Don't warn internal columns unless the user is in internal-column land. + col_names = col_names.filter((d) => !d.startsWith('_')); + } + throw new Error( + `Requested ${key} but table only has columns ["${col_names.join( + '", "', + )}]"`, + ); + } + } + + let column: Vector = tile.record_batch.getChild( + key[0], + ); + for (const k of key.slice(1)) { + column = column.getChild(k); + } + if (column.data.length !== 1) { + throw new Error(`Column ${key} has ${column.data.length} buffers, not 1.`); } + + if (!column.type || !column.type.typeId) { + throw new Error(`Column ${key} has no type.`); + } + + return column; } class MultipurposeBufferSet { @@ -1312,7 +1290,6 @@ class MultipurposeBufferSet { allocate_block(items: number, bytes_per_item: number): DS.BufferLocation { // Call dibs on a block of this buffer. // NB size is in **bytes** - const bytes_needed = items * bytes_per_item; let i = 0; for (const buffer_loc of this.freed_buffers) { @@ -1322,12 +1299,13 @@ class MultipurposeBufferSet { if (buffer_loc.byte_size === bytes_needed) { // Delete this element from the list of free buffers. this.freed_buffers.splice(i, 1); - return { + const v = { buffer: buffer_loc.buffer, offset: buffer_loc.offset, stride: bytes_per_item, byte_size: bytes_needed, }; + return v; } i += 1; } @@ -1344,7 +1322,9 @@ class MultipurposeBufferSet { offset: this.pointer, stride: bytes_per_item, byte_size: items * bytes_per_item, - } as DS.BufferLocation; + }; + + // add a listener for GC on the value. this.pointer += items * bytes_per_item; return value; } diff --git a/src/scatterplot.ts b/src/scatterplot.ts index 8980849e2..f3698dde5 100644 --- a/src/scatterplot.ts +++ b/src/scatterplot.ts @@ -389,10 +389,8 @@ export class Scatterplot { async reinitialize() { const { prefs } = this; - await this.deeptable.ready; await this.deeptable.root_tile.get_column('x'); - this._renderer = new ReglRenderer( '#container-for-webgl-canvas', this.deeptable, @@ -404,7 +402,6 @@ export class Scatterplot { this._zoom.initialize_zoom(); // Needs the zoom built as well. - const bkgd = select('#container-for-canvas-2d-background').select( 'canvas', ) as Selection< @@ -634,8 +631,9 @@ export class Scatterplot { return; } await this.plot_queue; + if (prefs) { - await this.start_transformations(prefs); + this.start_transformations(prefs); } this.plot_queue = this.unsafe_plotAPI(prefs); await this.plot_queue; @@ -672,43 +670,16 @@ export class Scatterplot { if (!prefs.encoding) { resolve(); } - const starttime = Date.now(); - if (this._renderer) { - const promises: Promise[] = []; - const sine_qua_non: Promise[] = [ - this.deeptable.root_tile.require_columns(needed_keys), - ]; - - // Immediately - for (const tile of this._renderer.visible_tiles()) { - // Allow unready tiles to stay unready; who know's what's going on there. - const manager = tile._buffer_manager; - if (manager !== undefined && manager.ready()) { - for (const key of needed_keys) { - if (tile.hasLoadedColumn(key)) { - manager.create_regl_buffer(key); - } - } - } - } - if (promises.length === 0) { - resolve(); - } else { - // It's important to get at least the first promise done, - // because it's used to determine some details about state. - void Promise.all(sine_qua_non).then(() => { - const endtime = Date.now(); - const elapsed = endtime - starttime; - if (elapsed < delay) { - setTimeout(() => { - resolve(); - }, delay - elapsed); - } else { - resolve(); - } - }); + this.deeptable.root_tile.require_columns(needed_keys); + // Immediately start loading what we can onto the GPUs, too. + for (const tile of this.renderer.visible_tiles()) { + this._renderer.bufferManager.ready( + tile, + [...needed_keys].map((k) => [k]), + ); } + resolve(); } else { resolve(); } @@ -772,11 +743,9 @@ export class Scatterplot { } } } - - if (this._zoom === undefined) { + if (this._zoom === undefined || this._renderer === undefined) { await this.reinitialize(); } - const renderer = this._renderer; const zoom = this._zoom; diff --git a/src/tile.ts b/src/tile.ts index c9b50511d..36b187d03 100644 --- a/src/tile.ts +++ b/src/tile.ts @@ -22,7 +22,6 @@ export type Rectangle = { // keys?: Array; // } -import type { TileBufferManager } from './regl_rendering'; import type { ArrowBuildable, LazyTileManifest, TileManifest } from './types'; import { isCompleteManifest } from './typing'; @@ -71,8 +70,6 @@ export class Tile { private arrowFetchCalls: Map = new Map(); public numeric_id: number; - // bindings to regl buffers holdings shadows of the RecordBatch. - public _buffer_manager?: TileBufferManager; //public child_locations: string[] = []; /** @@ -123,7 +120,8 @@ export class Tile { deleteColumn(colname: string) { if (this._batch) { - this._buffer_manager?.release(colname); + console.warn('Deleting column from tile doesnt free GPU memory'); + // this._buffer_manager?.release(colname); this._batch = add_or_delete_column(this.record_batch, colname, null); } // Ensure there is no cached version here. diff --git a/src/types.ts b/src/types.ts index 9b8c98f22..5bcd4ca6a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -19,11 +19,17 @@ import type { Buffer } from 'regl'; import type { DataSelection } from './selection'; import { Scatterplot } from './scatterplot'; import { ZoomTransform } from 'd3-zoom'; -import { TileBufferManager } from './regl_rendering'; import type { Tile } from './tile'; import type { Rectangle } from './tile'; export type { Renderer, Deeptable, ConcreteAesthetic }; +/** + * A struct that holds a buffer and information about where the GPU + * can find data on it. + * + * Note that the byte_size is *for the buffer*, and that individual elements + * may take views of it less than the byte_size. + */ export type BufferLocation = { buffer: Buffer; offset: number; @@ -640,8 +646,9 @@ export type GlobalDrawProps = { // Props that are needed to draw a single tile. export type TileDrawProps = GlobalDrawProps & { - manager: TileBufferManager; number: number; foreground_draw_number: 1 | 0 | -1; + tile: Tile; + count: number; tile_id: number; };