diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a7ea54..02cf0c3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v2.8.0 + +*20 oct 2022* + +- Updated patching.md file (#422) +- Minor documentation updates (#405) +- Added support to track vram usage (#395) +- Added texture compression support (#391) + ## v2.7.1 *19 sep 2022* @@ -333,3 +342,4 @@ ### Fixes * Fixed text rendering artifacts sometimes appearing on RPI platform (#41) + diff --git a/docs/RenderEngine/Textures/Toolbox.md b/docs/RenderEngine/Textures/Toolbox.md index 3ed851f7..915904fa 100644 --- a/docs/RenderEngine/Textures/Toolbox.md +++ b/docs/RenderEngine/Textures/Toolbox.md @@ -9,7 +9,7 @@ The `lng.Tools` Class contains useful functions for creating some commonly used |---|---| | Rounded rectangle | `lng.Tools.getRoundRect(w, h, radius, strokeWidth, strokeColor, fill, fillColor)` | | Drop-shadow rectangle | `lng.Tools.getShadowRect(w, h, radius = 0, blur = 5, margin = blur * 2)` | -| SVG rendering | `lng.Tools.createSvg(cb, stage, url, w, h)` | +| SVG rendering | `lng.Tools.getSvgTexture(url, w, h)` | ## Live Demo @@ -31,6 +31,12 @@ class TextureDemo extends lng.Application { zIndex: 1, color: 0x66000000, texture: lng.Tools.getShadowRect(150, 40, 4, 10, 15), + }, + Svg: { + x: 10, + y: 50, + zIndex: 0, + texture: lng.Tools.getSvgTexture(Utils.asset('images/image.svg'), 50, 50), } } } @@ -39,4 +45,4 @@ class TextureDemo extends lng.Application { const options = {stage: {w: window.innerWidth, h: window.innerHeight, useImageWorker: false}}; const App = new TextureDemo(options); document.body.appendChild(App.stage.getCanvas()); -``` \ No newline at end of file +``` diff --git a/docs/Transitions/Methods.md b/docs/Transitions/Methods.md index 7df3a291..d2bd3a90 100644 --- a/docs/Transitions/Methods.md +++ b/docs/Transitions/Methods.md @@ -26,7 +26,7 @@ To call a transition method, you have to get the correct transition `property` f ``` _init(){ - this.tag('MyObject').transition('x').start(); + this.tag('MyObject').transition('x').start(100); } ``` @@ -34,12 +34,11 @@ _init(){ | Name | Description | |---|---| -| `start()` | Start (or restart if already running) the transition | +| `start(targetValue : number)` | Start a new transition (similar to calling `setSmooth()`) | | `stop()` | Stop the transition (at the current value) | | `pause()` | Alias for `stop()` | | `play()` | Resume the paused transition | | `finish()` | Fast-forward the transition (to the target value) | -| `start(targetValue : number)` | Start a new transition (similar to calling `setSmooth()`) | | `reset(targetValue : number, p : number)` | Set the transition at a specific point in time (where p is a value between 0 and 1) | | `updateTargetValue(targetValue : number)` | Update the target value while keeping the current progress and value | @@ -124,4 +123,4 @@ class BasicUsageExample extends lng.Application { const options = {stage: {w: window.innerWidth, h: window.innerHeight, useImageWorker: false}}; const App = new BasicUsageExample(options); document.body.appendChild(App.stage.getCanvas()); -``` \ No newline at end of file +``` diff --git a/examples/texture-compression/lightning-etc1.pvr b/examples/texture-compression/lightning-etc1.pvr new file mode 100644 index 00000000..79cbcfce Binary files /dev/null and b/examples/texture-compression/lightning-etc1.pvr differ diff --git a/examples/texture-compression/sampleImage-astc_4x4.ktx b/examples/texture-compression/sampleImage-astc_4x4.ktx new file mode 100644 index 00000000..75b931bc Binary files /dev/null and b/examples/texture-compression/sampleImage-astc_4x4.ktx differ diff --git a/examples/texture-compression/sampleImage-etc1.ktx b/examples/texture-compression/sampleImage-etc1.ktx new file mode 100644 index 00000000..03cf50ed Binary files /dev/null and b/examples/texture-compression/sampleImage-etc1.ktx differ diff --git a/examples/texture-compression/sampleImage-pvrtc.ktx b/examples/texture-compression/sampleImage-pvrtc.ktx new file mode 100644 index 00000000..86ed8e66 Binary files /dev/null and b/examples/texture-compression/sampleImage-pvrtc.ktx differ diff --git a/examples/texture-compression/sampleImage-s3tc.ktx b/examples/texture-compression/sampleImage-s3tc.ktx new file mode 100644 index 00000000..0f52d610 Binary files /dev/null and b/examples/texture-compression/sampleImage-s3tc.ktx differ diff --git a/examples/texture-compression/texture-compression.html b/examples/texture-compression/texture-compression.html new file mode 100644 index 00000000..9d0ee5a4 --- /dev/null +++ b/examples/texture-compression/texture-compression.html @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index d5168fce..9a1da913 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lightningjs/core", - "version": "2.7.1", + "version": "2.8.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@lightningjs/core", - "version": "2.7.1", + "version": "2.8.0", "license": "Apache-2.0", "devDependencies": { "@babel/core": "^7.8.3", diff --git a/package.json b/package.json index 3fea9273..ede31b19 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "author": "Metrological, Bas van Meurs ", "name": "@lightningjs/core", - "version": "2.7.1", + "version": "2.8.0", "license": "Apache-2.0", "main": "dist/lightning.js", "module": "index.js", diff --git a/src/platforms/browser/WebPlatform.mjs b/src/platforms/browser/WebPlatform.mjs index 393f9e81..a7a588e5 100644 --- a/src/platforms/browser/WebPlatform.mjs +++ b/src/platforms/browser/WebPlatform.mjs @@ -90,7 +90,7 @@ export default class WebPlatform { loop() { let self = this; - let lp = function() { + let lp = function () { self._awaitingLoop = false; if (self._looping) { self.stage.updateFrame(); @@ -105,6 +105,23 @@ export default class WebPlatform { requestAnimationFrame(lp); } + uploadCompressedGlTexture(gl, textureSource, source, options) { + const view = !source.pvr ? new DataView(source.mipmaps[0]) : source.mipmaps[0]; + gl.compressedTexImage2D( + gl.TEXTURE_2D, + 0, + source.glInternalFormat, + source.pixelWidth, + source.pixelHeight, + 0, + view, + ) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + } + uploadGlTexture(gl, textureSource, source, options) { if (source instanceof ImageData || source instanceof HTMLImageElement || source instanceof HTMLVideoElement || (window.ImageBitmap && source instanceof ImageBitmap)) { // Web-specific data types. @@ -124,24 +141,155 @@ export default class WebPlatform { } } - loadSrcTexture({src, hasAlpha}, cb) { + /** + * KTX File format specification + * https://www.khronos.org/registry/KTX/specs/1.0/ktxspec_v1.html + **/ + handleKtxLoad(cb, src) { + var self = this; + return function () { + var arraybuffer = this.response; + var view = new DataView(arraybuffer); + + // identifier, big endian + var targetIdentifier = 3632701469 + if (targetIdentifier !== (view.getUint32(0) + view.getUint32(4) + view.getUint32(8))) { + cb('Parsing failed: identifier ktx mismatch:', src) + } + + var littleEndian = (view.getUint32(12) === 16909060) ? true : false; + var data = { + glType: view.getUint32(16, littleEndian), + glTypeSize: view.getUint32(20, littleEndian), + glFormat: view.getUint32(24, littleEndian), + glInternalFormat: view.getUint32(28, littleEndian), + glBaseInternalFormat: view.getUint32(32, littleEndian), + pixelWidth: view.getUint32(36, littleEndian), + pixelHeight: view.getUint32(40, littleEndian), + pixelDepth: view.getUint32(44, littleEndian), + numberOfArrayElements: view.getUint32(48, littleEndian), + numberOfFaces: view.getUint32(52, littleEndian), + numberOfMipmapLevels: view.getUint32(56, littleEndian), + bytesOfKeyValueData: view.getUint32(60, littleEndian), + kvps: [], + mipmaps: [], + get width() { return this.pixelWidth }, + get height() { return this.pixelHeight }, + }; + + const props = (obj) => { + const p = []; + for (let v in obj) { + p.push(obj[v]); + } + return p; + } + + const formats = Object.values(self.stage.renderer.getCompressedTextureExtensions()) + .filter((obj) => obj != null) + .map((obj) => props(obj)) + .reduce((prev, current) => prev.concat(current)); + + if (!formats.includes(data.glInternalFormat)) { + console.warn("[Lightning] Unrecognized texture extension format:", src, data.glInternalFormat, self.stage.renderer.getCompressedTextureExtensions()); + } + + var offset = 64 + // Key Value Pairs of data start at byte offset 64 + // But the only known kvp is the API version, so skipping parsing. + offset += data.bytesOfKeyValueData; + + for (var i = 0; i < data.numberOfMipmapLevels; i++) { + var imageSize = view.getUint32(offset); + offset += 4; + data.mipmaps.push(view.buffer.slice(offset, imageSize)); + offset += imageSize + } + + cb(null, { + source: data, + renderInfo: { src: src, compressed: true }, + }) + } + } + + handlePvrLoad(cb, src) { + return function () { + // pvr header length in 32 bits + const pvrHeaderLength = 13; + // for now only we only support: COMPRESSED_RGB_ETC1_WEBGL + const pvrFormatEtc1 = 0x8D64; + const pvrWidth = 7; + const pvrHeight = 6; + const pvrMipmapCount = 11; + const pvrMetadata = 12; + const arrayBuffer = this.response; + const header = new Int32Array(arrayBuffer, 0, pvrHeaderLength); + const dataOffset = header[pvrMetadata] + 52; + const pvrtcData = new Uint8Array(arrayBuffer, dataOffset); + + var data = { + glInternalFormat: pvrFormatEtc1, + pixelWidth: header[pvrWidth], + pixelHeight: header[pvrHeight], + numberOfMipmapLevels: header[pvrMipmapCount], + mipmaps: [], + pvr: true, + get width() { return this.pixelWidth }, + get height() { return this.pixelHeight }, + }; + + let offset = 0 + let width = data.pixelWidth; + let height = data.pixelHeight; + + for (var i = 0; i < data.numberOfMipmapLevels; i++) { + const level = ((width + 3) >> 2) * ((height + 3) >> 2) * 8; + const view = new Uint8Array(arrayBuffer, pvrtcData.byteOffset + offset, level); + data.mipmaps.push(view); + offset += level; + width = width >> 1; + height = height >> 1; + } + + cb(null, { + source: data, + renderInfo: { src: src, compressed: true }, + }) + } + } + + loadSrcTexture({ src, hasAlpha }, cb) { let cancelCb = undefined; let isPng = (src.indexOf(".png") >= 0) || src.substr(0, 21) == 'data:image/png;base64'; - if (this._imageWorker) { + let isKtx = src.indexOf('.ktx') >= 0; + let isPvr = src.indexOf('.pvr') >= 0; + if (isKtx || isPvr) { + let request = new XMLHttpRequest(); + request.addEventListener( + "load", isKtx ? this.handleKtxLoad(cb, src) : this.handlePvrLoad(cb, src) + ); + request.open("GET", src); + request.responseType = "arraybuffer"; + request.send(); + cancelCb = function () { + request.abort(); + } + } else if (this._imageWorker) { // WPE-specific image parser. const image = this._imageWorker.create(src); - image.onError = function(err) { + image.onError = function (err) { return cb("Image load error"); }; - image.onLoad = function({imageBitmap, hasAlphaChannel}) { + image.onLoad = function ({ imageBitmap, hasAlphaChannel }) { cb(null, { source: imageBitmap, - renderInfo: {src: src}, + renderInfo: { src: src, compressed: false }, hasAlpha: hasAlphaChannel, premultiplyAlpha: true }); }; - cancelCb = function() { + cancelCb = function () { image.cancel(); } } else { @@ -149,26 +297,26 @@ export default class WebPlatform { // On the PS4 platform setting the `crossOrigin` attribute on // images can cause CORS failures. - if (!(src.substr(0,5) == "data:") && !Utils.isPS4) { + if (!(src.substr(0, 5) == "data:") && !Utils.isPS4) { // Base64. image.crossOrigin = "Anonymous"; } - image.onerror = function(err) { + image.onerror = function (err) { // Ignore error message when cancelled. if (image.src) { return cb("Image load error"); } }; - image.onload = function() { + image.onload = function () { cb(null, { source: image, - renderInfo: {src: src}, + renderInfo: { src: src, compressed: false }, hasAlpha: isPng || hasAlpha }); }; image.src = src; - cancelCb = function() { + cancelCb = function () { image.onerror = null; image.onload = null; image.removeAttribute('src'); diff --git a/src/renderer/webgl/WebGLRenderer.d.mts b/src/renderer/webgl/WebGLRenderer.d.mts index 93bf1c68..d09fdc01 100644 --- a/src/renderer/webgl/WebGLRenderer.d.mts +++ b/src/renderer/webgl/WebGLRenderer.d.mts @@ -21,4 +21,5 @@ import Renderer from "../Renderer.mjs"; export default class WebGLRenderer extends Renderer { constructor(stage: Stage); + getCompressedTextureExtensions(): Object; } diff --git a/src/renderer/webgl/WebGLRenderer.mjs b/src/renderer/webgl/WebGLRenderer.mjs index 247f874e..0e0728ea 100644 --- a/src/renderer/webgl/WebGLRenderer.mjs +++ b/src/renderer/webgl/WebGLRenderer.mjs @@ -32,6 +32,16 @@ export default class WebGLRenderer extends Renderer { constructor(stage) { super(stage); this.shaderPrograms = new Map(); + this._compressedTextureExtensions = { + astc: stage.gl.getExtension('WEBGL_compressed_texture_astc'), + etc1: stage.gl.getExtension('WEBGL_compressed_texture_etc1'), + s3tc: stage.gl.getExtension('WEBGL_compressed_texture_s3tc'), + pvrtc: stage.gl.getExtension('WEBGL_compressed_texture_pvrtc'), + } + } + + getCompressedTextureExtensions() { + return this._compressedTextureExtensions } destroy() { @@ -61,7 +71,7 @@ export default class WebGLRenderer extends Renderer { createCoreRenderExecutor(ctx) { return new WebGLCoreRenderExecutor(ctx); } - + createCoreRenderState(ctx) { return new CoreRenderState(ctx); } @@ -83,28 +93,67 @@ export default class WebGLRenderer extends Renderer { glTexture.params[gl.TEXTURE_MIN_FILTER] = gl.LINEAR; glTexture.params[gl.TEXTURE_WRAP_S] = gl.CLAMP_TO_EDGE; glTexture.params[gl.TEXTURE_WRAP_T] = gl.CLAMP_TO_EDGE; - glTexture.options = {format: gl.RGBA, internalFormat: gl.RGBA, type: gl.UNSIGNED_BYTE}; + glTexture.options = { format: gl.RGBA, internalFormat: gl.RGBA, type: gl.UNSIGNED_BYTE }; // We need a specific framebuffer for every render texture. glTexture.framebuffer = gl.createFramebuffer(); - glTexture.projection = new Float32Array([2/w, 2/h]); + glTexture.projection = new Float32Array([2 / w, 2 / h]); gl.bindFramebuffer(gl.FRAMEBUFFER, glTexture.framebuffer); gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, glTexture, 0); return glTexture; } - + freeRenderTexture(glTexture) { let gl = this.stage.gl; gl.deleteFramebuffer(glTexture.framebuffer); gl.deleteTexture(glTexture); } + _getBytesPerPixel(fmt, type) { + const gl = this.stage.gl; + + if (fmt === gl.RGBA) { + switch (type) { + case gl.UNSIGNED_BYTE: + return 4; + + case gl.UNSIGNED_SHORT_4_4_4_4: + return 2; + + case gl.UNSIGNED_SHORT_5_5_5_1: + return 2; + + default: + throw new Error('Invalid type specified for GL_RGBA format'); + } + } + else if (fmt === gl.RGB) { + switch (type) { + case gl.UNSIGNED_BYTE: + return 3; + + case gl.UNSIGNED_BYTE_5_6_5: + return 2; + + default: + throw new Error('Invalid type specified for GL_RGB format'); + } + } + else { + throw new Error('Invalid format specified in call to _getBytesPerPixel()'); + } + } + uploadTextureSource(textureSource, options) { const gl = this.stage.gl; const source = options.source; + let compressed = false; + if (options.renderInfo) { + compressed = options.renderInfo.compressed || false + } const format = { premultiplyAlpha: true, @@ -150,6 +199,11 @@ export default class WebGLRenderer extends Renderer { gl.texParameteri(gl.TEXTURE_2D, parseInt(key), value); }); + if (compressed) { + this.stage.platform.uploadCompressedGlTexture(gl, textureSource, source); + return glTexture; + } + const texOptions = format.texOptions; texOptions.format = texOptions.format || (format.hasAlpha ? gl.RGBA : gl.RGB); texOptions.type = texOptions.type || gl.UNSIGNED_BYTE; @@ -157,12 +211,15 @@ export default class WebGLRenderer extends Renderer { if (options && options.imageRef) { texOptions.imageRef = options.imageRef; } - + this.stage.platform.uploadGlTexture(gl, textureSource, source, texOptions); - + glTexture.params = Utils.cloneObjShallow(texParams); glTexture.options = Utils.cloneObjShallow(texOptions); + // calculate bytes per pixel for vram usage tracking + glTexture.bytesPerPixel = this._getBytesPerPixel(texOptions.format, texOptions.type); + return glTexture; } diff --git a/src/tree/Stage.d.mts b/src/tree/Stage.d.mts index acd5feb4..2c90f287 100644 --- a/src/tree/Stage.d.mts +++ b/src/tree/Stage.d.mts @@ -535,10 +535,25 @@ declare class Stage extends EventEmitter { // addMemoryUsage(delta: any): void; // - Internal use only. /** - * Amount of texture memory used (in bytes) + * Amount of memory used (in pixels) */ get usedMemory(): number; + /** + * Amount of memory used by textures (in bytes) + */ + get usedVram(): number; + + /** + * Amount of memory used by textures with alpha channel (in bytes) + */ + get usedVramAlpha(): number; + + /** + * Amount of memory used by textures without alpha channel (in bytes) + */ + get usedVramNonAlpha(): number; + /** * Runs the texture memory garbage collector. * diff --git a/src/tree/Stage.mjs b/src/tree/Stage.mjs index aee49021..7457f685 100644 --- a/src/tree/Stage.mjs +++ b/src/tree/Stage.mjs @@ -39,6 +39,10 @@ export default class Stage extends EventEmitter { this._usedMemory = 0; this._lastGcFrame = 0; + // attempt to track VRAM usage more accurately by accounting for different color channels + this._usedVramAlpha = 0; + this._usedVramNonAlpha = 0; + const platformType = Stage.platform ? Stage.platform : PlatformLoader.load(options); this.platform = new platformType(); @@ -419,6 +423,27 @@ export default class Stage extends EventEmitter { return this._usedMemory; } + addVramUsage(delta, alpha) { + if (alpha) { + this._usedVramAlpha += delta; + } + else { + this._usedVramNonAlpha += delta; + } + } + + get usedVramAlpha() { + return this._usedVramAlpha; + } + + get usedVramNonAlpha() { + return this._usedVramNonAlpha; + } + + get usedVram() { + return this._usedVramAlpha + this._usedVramNonAlpha; + } + gc(aggressive) { if (this._lastGcFrame !== this.frameCounter) { this._lastGcFrame = this.frameCounter; diff --git a/src/tree/TextureManager.mjs b/src/tree/TextureManager.mjs index 67dd589b..c170f9bc 100644 --- a/src/tree/TextureManager.mjs +++ b/src/tree/TextureManager.mjs @@ -18,6 +18,7 @@ */ import TextureSource from "./TextureSource.mjs"; +import Stage from './Stage.mjs'; export default class TextureManager { @@ -96,13 +97,37 @@ export default class TextureManager { this._uploadedTextureSources.push(textureSource); this.addToLookupMap(textureSource); + + // add VRAM tracking if using the webgl renderer + this._updateVramUsage(textureSource, 1); } _addMemoryUsage(delta) { this._usedMemory += delta; this.stage.addMemoryUsage(delta); } - + + _updateVramUsage(textureSource, sign) { + const nativeTexture = textureSource.nativeTexture; + var usage; + + // do nothing if webgl isn't even supported + if (!Stage.isWebglSupported()) + return; + + // or if there is no native texture + if (!textureSource.isLoaded()) + return; + + // or, finally, if there is no bytes per pixel specified + if (!nativeTexture.hasOwnProperty('bytesPerPixel') || isNaN(nativeTexture.bytesPerPixel)) + return; + + usage = sign * (textureSource.w * textureSource.h * nativeTexture.bytesPerPixel); + + this.stage.addVramUsage(usage, textureSource.hasAlpha); + } + addToLookupMap(textureSource) { const lookupId = textureSource.lookupId; if (lookupId) { @@ -137,6 +162,9 @@ export default class TextureManager { if (textureSource.isLoaded()) { this._nativeFreeTextureSource(textureSource); this._addMemoryUsage(-textureSource.w * textureSource.h); + + // add VRAM tracking if using the webgl renderer + this._updateVramUsage(textureSource, -1); } // Should be reloaded. diff --git a/src/tree/TextureSource.mjs b/src/tree/TextureSource.mjs index 1afcac9a..17268611 100644 --- a/src/tree/TextureSource.mjs +++ b/src/tree/TextureSource.mjs @@ -102,6 +102,17 @@ export default class TextureSource { */ this._imageRef = null; + + /** + * Track whether or not there is an alpha channel in this source + * @type {boolean} + * @private + */ + this._hasAlpha = false; + } + + get hasAlpha() { + return this._hasAlpha; } get loadError() { @@ -256,6 +267,7 @@ export default class TextureSource { setSource(options) { const source = options.source; + this._hasAlpha = (options ? (options.hasAlpha || false) : false); this.w = source.width || (options && options.w) || 0; this.h = source.height || (options && options.h) || 0;