From 4c39a6503cb99ff7cdec8917714b02de73515a6b Mon Sep 17 00:00:00 2001 From: Kieran Farr Date: Fri, 15 Dec 2023 14:21:08 -0800 Subject: [PATCH] next round of splat component updates --- lib/aframe-gaussian-splatting-component.min.js | 2 +- lib/aframe-gaussian-splatting-component.min.js.map | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/aframe-gaussian-splatting-component.min.js b/lib/aframe-gaussian-splatting-component.min.js index 41d3342..c4afc71 100644 --- a/lib/aframe-gaussian-splatting-component.min.js +++ b/lib/aframe-gaussian-splatting-component.min.js @@ -101,4 +101,4 @@ AFRAME.registerComponent("gaussian_splatting",{schema:{src:{type:"string",defaul if(B < fDF) discard; gl_FragColor = vec4(vColor.rgb, B); } - `,blending:THREE.CustomBlending,blendSrcAlpha:THREE.OneFactor,depthTest:!0,depthWrite:this.data.depthWrite,transparent:!0}),mesh=(material.onBeforeRender=(renderer,scene,camera,geometry,object,group)=>{var projectionMatrix=this.getProjectionMatrix(camera),camera=(mesh.material.uniforms.gsProjectionMatrix.value=projectionMatrix,mesh.material.uniforms.gsModelViewMatrix.value=this.getModelViewMatrix(camera),new THREE.Vector4),renderer=(renderer.getCurrentViewport(camera),camera.w/2*Math.abs(projectionMatrix.elements[5]));material.uniforms.viewport.value[0]=camera.z,material.uniforms.viewport.value[1]=camera.w,material.uniforms.focal.value=renderer},new THREE.Mesh(positionsArray,material));for(mesh.frustumCulled=!1,this.object.add(mesh),this.worker.onmessage=e=>{e=new Uint32Array(e.data.sortedIndexes);mesh.geometry.attributes.splatIndex.set(e),mesh.geometry.attributes.splatIndex.needsUpdate=!0,mesh.geometry.instanceCount=e.length,this.sortReady=!0};;){var centerAndScaleTextureProperties=this.renderer.properties.get(this.centerAndScaleTexture),covAndColorTextureProperties=this.renderer.properties.get(this.covAndColorTexture);if(centerAndScaleTextureProperties&¢erAndScaleTextureProperties.__webglTexture&&covAndColorTextureProperties&¢erAndScaleTextureProperties.__webglTexture)break;await new Promise(resolve=>setTimeout(resolve,10))}this.sortReady=!0},loadData:function(camera,object,renderer,src){this.camera=camera,this.object=object,this.renderer=renderer,this.loadedVertexCount=0,this.rowLength=32,this.worker=new Worker(URL.createObjectURL(new Blob(["(",this.createWorker.toString(),")(self)"],{type:"application/javascript"}))),this.worker.postMessage({method:"clear"}),fetch(src).then(async data=>{console.log(data);var reader=data.body.getReader();let glInitialized=!1,bytesDownloaded=0,bytesProcesses=0;var data=data.headers.get("Content-Length"),totalDownloadBytes=data?parseInt(data):void 0,chunks=(null!=totalDownloadBytes&&(data=Math.floor(totalDownloadBytes/this.rowLength),await this.initGL(data),glInitialized=!0),[]),start=Date.now();let lastReportedProgress=0;for(var isPly=src.endsWith(".ply");;)try{var mbps,percent,{value,done}=await reader.read();if(done){console.log("Completed download.");break}bytesDownloaded+=value.length,null!=totalDownloadBytes?(mbps=bytesDownloaded/1024/1024/((Date.now()-start)/1e3),1<(percent=bytesDownloaded/totalDownloadBytes*100)-lastReportedProgress&&(console.log("download progress:",percent.toFixed(2)+"%",mbps.toFixed(2)+" Mbps"),lastReportedProgress=percent)):console.log("download progress:",bytesDownloaded,", unknown total"),chunks.push(value);var bytesRemains=bytesDownloaded-bytesProcesses;if(!isPly&&null!=totalDownloadBytes&&bytesRemains>this.rowLength){var extra_data,vertexCount=Math.floor(bytesRemains/this.rowLength),concatenatedChunksbuffer=new Uint8Array(bytesRemains);let offset=0;for(const chunk of chunks)concatenatedChunksbuffer.set(chunk,offset),offset+=chunk.length;chunks.length=0,bytesRemains>vertexCount*this.rowLength&&((extra_data=new Uint8Array(bytesRemains-vertexCount*this.rowLength)).set(concatenatedChunksbuffer.subarray(bytesRemains-extra_data.length,bytesRemains),0),chunks.push(extra_data));var buffer=new Uint8Array(vertexCount*this.rowLength);buffer.set(concatenatedChunksbuffer.subarray(0,buffer.byteLength),0),this.pushDataBuffer(buffer.buffer,vertexCount),bytesProcesses+=vertexCount*this.rowLength}}catch(error){console.error(error);break}if(0acc+chunk.length,0)),offset=0;for(const chunk of chunks)concatenatedChunks.set(chunk,offset),offset+=chunk.length;isPly&&(concatenatedChunks=new Uint8Array(this.processPlyBuffer(concatenatedChunks.buffer)));let numVertexes=Math.floor(concatenatedChunks.byteLength/this.rowLength);glInitialized||(await this.initGL(numVertexes),glInitialized=!0),this.pushDataBuffer(concatenatedChunks.buffer,numVertexes)}}).catch(error=>{throw error})},pushDataBuffer:function(buffer,vertexCount){if(this.loadedVertexCount+vertexCount>this.maxVertexes&&(console.log("vertexCount limited to ",this.maxVertexes,vertexCount),vertexCount=this.maxVertexes-this.loadedVertexCount),!(vertexCount<=0)){var u_buffer=new Uint8Array(buffer),f_buffer=new Float32Array(buffer),matrices=new Float32Array(16*vertexCount),covAndColorData_uint8=new Uint8Array(this.covAndColorData.buffer),covAndColorData_int16=new Int16Array(this.covAndColorData.buffer);for(let i=0;imax_value&&(max_value=Math.abs(mtx.elements[cov_indexes[j]]));let destOffset=4*this.loadedVertexCount+4*i;this.centerAndScaleData[destOffset+0]=center.x,this.centerAndScaleData[destOffset+1]=center.y,this.centerAndScaleData[destOffset+2]=center.z,this.centerAndScaleData[destOffset+3]=max_value/32767,destOffset=8*this.loadedVertexCount+4*i*2;for(let j=0;j-1e-4*depth&&cutoutArea&&(depthList[validCount]=depth,validIndexList[validCount]=i,validCount++,depth>maxDepth&&(maxDepth=depth),depth{if("clear"==e.data.method&&(matrices=void 0),"push"==e.data.method&&(new_matrices=new Float32Array(e.data.matrices),matrices=void 0===matrices?new_matrices:((resized=new Float32Array(matrices.length+new_matrices.length)).set(matrices),resized.set(new_matrices,matrices.length),resized)),"sort"==e.data.method)if(void 0===matrices){var new_matrices=new Uint32Array(1);self.postMessage({sortedIndexes:new_matrices},[new_matrices.buffer])}else{new_matrices=new Float32Array(e.data.view),e=void 0!==e.data.cutout?new Float32Array(e.data.cutout):void 0;const sortedIndexes=sortSplats(matrices,new_matrices,e);self.postMessage({sortedIndexes:sortedIndexes},[sortedIndexes.buffer])}}},processPlyBuffer:function(inputBuffer){var ubuf=new Uint8Array(inputBuffer),ubuf=(new TextDecoder).decode(ubuf.slice(0,10240)),header_end_index=ubuf.indexOf("end_header\n");if(header_end_index<0)throw new Error("Unable to read .ply file header");var vertexCount=parseInt(/element vertex (\d+)\n/.exec(ubuf)[1]);console.log("Vertex Count",vertexCount);let row_offset=0,offsets={},types={};var prop,TYPE_MAP={double:"getFloat64",int:"getInt32",uint:"getUint32",float:"getFloat32",short:"getInt16",ushort:"getUint16",uchar:"getUint8"};for(prop of ubuf.slice(0,header_end_index).split("\n").filter(k=>k.startsWith("property "))){var[,type,name]=prop.split(" "),type=TYPE_MAP[type]||"getInt8";types[name]=type,offsets[name]=row_offset,row_offset+=parseInt(type.replace(/[^\d]/g,""))/8}console.log("Bytes per row",row_offset,types,offsets);let dataView=new DataView(inputBuffer,header_end_index+"end_header\n".length),row=0;var attrs=new Proxy({},{get(target,prop){if(types[prop])return dataView[types[prop]](row*row_offset+offsets[prop],!0);throw new Error(prop+" not found")}});console.time("calculate importance");let sizeList=new Float32Array(vertexCount);var size,opacity,sizeIndex=new Uint32Array(vertexCount);for(row=0;rowsizeList[a]-sizeList[b]),console.timeEnd("sort");var buffer=new ArrayBuffer(32*vertexCount);console.time("build buffer");for(let j=0;j{var projectionMatrix=this.getProjectionMatrix(camera),camera=(mesh.material.uniforms.gsProjectionMatrix.value=projectionMatrix,mesh.material.uniforms.gsModelViewMatrix.value=this.getModelViewMatrix(camera),new THREE.Vector4),renderer=(renderer.getCurrentViewport(camera),camera.w/2*Math.abs(projectionMatrix.elements[5]));material.uniforms.viewport.value[0]=camera.z,material.uniforms.viewport.value[1]=camera.w,material.uniforms.focal.value=renderer},new THREE.Mesh(positionsArray,material));for(mesh.frustumCulled=!1,this.object.add(mesh),this.worker.onmessage=e=>{e=new Uint32Array(e.data.sortedIndexes);mesh.geometry.attributes.splatIndex.set(e),mesh.geometry.attributes.splatIndex.needsUpdate=!0,mesh.geometry.instanceCount=e.length,this.sortReady=!0};;){var centerAndScaleTextureProperties=this.renderer.properties.get(this.centerAndScaleTexture),covAndColorTextureProperties=this.renderer.properties.get(this.covAndColorTexture);if(centerAndScaleTextureProperties&¢erAndScaleTextureProperties.__webglTexture&&covAndColorTextureProperties&¢erAndScaleTextureProperties.__webglTexture)break;await new Promise(resolve=>setTimeout(resolve,10))}this.sortReady=!0},loadData:async function(camera,object,renderer,src){this.camera=camera,this.object=object,this.renderer=renderer,this.loadedVertexCount=0,this.rowLength=32,this.worker=new Worker(URL.createObjectURL(new Blob(["(",this.createWorker.toString(),")(self)"],{type:"application/javascript"}))),this.worker.postMessage({method:"clear"});try{for(;;){if((data=await fetch(src)).ok)break;console.log("retry",src),await new Promise(resolve=>setTimeout(resolve,1e3))}console.log(data),this.parseData(data,src)}catch(error){console.error(error)}},parseData:async function(data,src){var reader=data.body.getReader();let glInitialized=!1,bytesDownloaded=0,bytesProcesses=0;var data=data.headers.get("Content-Length"),totalDownloadBytes=data?parseInt(data):void 0,chunks=(null!=totalDownloadBytes&&(data=Math.floor(totalDownloadBytes/this.rowLength),await this.initGL(data),glInitialized=!0),[]),start=Date.now();let lastReportedProgress=0;for(var isPly=src.endsWith(".ply");;)try{var mbps,percent,{value,done}=await reader.read();if(done){console.log("Completed download.");break}bytesDownloaded+=value.length,null!=totalDownloadBytes?(mbps=bytesDownloaded/1024/1024/((Date.now()-start)/1e3),1<(percent=bytesDownloaded/totalDownloadBytes*100)-lastReportedProgress&&(console.log("download progress:",percent.toFixed(2)+"%",mbps.toFixed(2)+" Mbps"),lastReportedProgress=percent)):console.log("download progress:",bytesDownloaded,", unknown total"),chunks.push(value);var bytesRemains=bytesDownloaded-bytesProcesses;if(!isPly&&null!=totalDownloadBytes&&bytesRemains>this.rowLength){var extra_data,vertexCount=Math.floor(bytesRemains/this.rowLength),concatenatedChunksbuffer=new Uint8Array(bytesRemains);let offset=0;for(const chunk of chunks)concatenatedChunksbuffer.set(chunk,offset),offset+=chunk.length;chunks.length=0,bytesRemains>vertexCount*this.rowLength&&((extra_data=new Uint8Array(bytesRemains-vertexCount*this.rowLength)).set(concatenatedChunksbuffer.subarray(bytesRemains-extra_data.length,bytesRemains),0),chunks.push(extra_data));var buffer=new Uint8Array(vertexCount*this.rowLength);buffer.set(concatenatedChunksbuffer.subarray(0,buffer.byteLength),0),this.pushDataBuffer(buffer.buffer,vertexCount),bytesProcesses+=vertexCount*this.rowLength}}catch(error){console.error(error);break}if(0acc+chunk.length,0)),offset=0;for(const chunk of chunks)concatenatedChunks.set(chunk,offset),offset+=chunk.length;isPly&&(concatenatedChunks=new Uint8Array(this.processPlyBuffer(concatenatedChunks.buffer)));let numVertexes=Math.floor(concatenatedChunks.byteLength/this.rowLength);glInitialized||(await this.initGL(numVertexes),glInitialized=!0),this.pushDataBuffer(concatenatedChunks.buffer,numVertexes)}},pushDataBuffer:function(buffer,vertexCount){if(this.loadedVertexCount+vertexCount>this.maxVertexes&&(console.log("vertexCount limited to ",this.maxVertexes,vertexCount),vertexCount=this.maxVertexes-this.loadedVertexCount),!(vertexCount<=0)){var u_buffer=new Uint8Array(buffer),f_buffer=new Float32Array(buffer),matrices=new Float32Array(16*vertexCount),covAndColorData_uint8=new Uint8Array(this.covAndColorData.buffer),covAndColorData_int16=new Int16Array(this.covAndColorData.buffer);for(let i=0;imax_value&&(max_value=Math.abs(mtx.elements[cov_indexes[j]]));let destOffset=4*this.loadedVertexCount+4*i;this.centerAndScaleData[destOffset+0]=center.x,this.centerAndScaleData[destOffset+1]=center.y,this.centerAndScaleData[destOffset+2]=center.z,this.centerAndScaleData[destOffset+3]=max_value/32767,destOffset=8*this.loadedVertexCount+4*i*2;for(let j=0;j-1e-4*depth&&cutoutArea&&(depthList[validCount]=depth,validIndexList[validCount]=i,validCount++,depth>maxDepth&&(maxDepth=depth),depth{if("clear"==e.data.method&&(matrices=void 0),"push"==e.data.method&&(new_matrices=new Float32Array(e.data.matrices),matrices=void 0===matrices?new_matrices:((resized=new Float32Array(matrices.length+new_matrices.length)).set(matrices),resized.set(new_matrices,matrices.length),resized)),"sort"==e.data.method)if(void 0===matrices){var new_matrices=new Uint32Array(1);self.postMessage({sortedIndexes:new_matrices},[new_matrices.buffer])}else{new_matrices=new Float32Array(e.data.view),e=void 0!==e.data.cutout?new Float32Array(e.data.cutout):void 0;const sortedIndexes=sortSplats(matrices,new_matrices,e);self.postMessage({sortedIndexes:sortedIndexes},[sortedIndexes.buffer])}}},processPlyBuffer:function(inputBuffer){var ubuf=new Uint8Array(inputBuffer),ubuf=(new TextDecoder).decode(ubuf.slice(0,10240)),header_end_index=ubuf.indexOf("end_header\n");if(header_end_index<0)throw new Error("Unable to read .ply file header");var vertexCount=parseInt(/element vertex (\d+)\n/.exec(ubuf)[1]);console.log("Vertex Count",vertexCount);let row_offset=0,offsets={},types={};var prop,TYPE_MAP={double:"getFloat64",int:"getInt32",uint:"getUint32",float:"getFloat32",short:"getInt16",ushort:"getUint16",uchar:"getUint8"};for(prop of ubuf.slice(0,header_end_index).split("\n").filter(k=>k.startsWith("property "))){var[,type,name]=prop.split(" "),type=TYPE_MAP[type]||"getInt8";types[name]=type,offsets[name]=row_offset,row_offset+=parseInt(type.replace(/[^\d]/g,""))/8}console.log("Bytes per row",row_offset,types,offsets);let dataView=new DataView(inputBuffer,header_end_index+"end_header\n".length),row=0;var attrs=new Proxy({},{get(target,prop){if(types[prop])return dataView[types[prop]](row*row_offset+offsets[prop],!0);throw new Error(prop+" not found")}});console.time("calculate importance");let sizeList=new Float32Array(vertexCount);var size,opacity,sizeIndex=new Uint32Array(vertexCount);for(row=0;rowsizeList[a]-sizeList[b]),console.timeEnd("sort");var buffer=new ArrayBuffer(32*vertexCount);console.time("build buffer");for(let j=0;j