Merge pull request #267 from open-mmlab/dev

Apply pointCloud's feature

package.json

@@ -26,23 +26,23 @@
   "devDependencies": {
     "@commitlint/cli": "8.3.5",
     "@commitlint/config-conventional": "8.3.4",
-    "@octokit/rest": "^19.0.7",
-    "@semantic-release/commit-analyzer": "^9.0.2",
-    "@semantic-release/exec": "^6.0.3",
-    "@semantic-release/git": "^10.0.1",
-    "@semantic-release/github": "^8.0.7",
-    "@semantic-release/npm": "^10.0.3",
-    "@semantic-release/release-notes-generator": "^10.0.3",
-    "commitizen": "^4.2.4",
-    "cz-conventional-changelog": "^3.3.0",
-    "husky": "^4.0.3",
-    "lerna": "^4.0.0",
-    "minimist": "^1.2.8",
-    "multi-semantic-release": "^3.0.2",
-    "node-fetch": "^2.6.1",
-    "semantic-release": "^21.0.1",
-    "semantic-release-github-pullrequest": "^1.3.0",
-    "standard-version": "^9.3.1"
+    "@octokit/rest": "19.0.7",
+    "@semantic-release/commit-analyzer": "9.0.2",
+    "@semantic-release/exec": "6.0.3",
+    "@semantic-release/git": "10.0.1",
+    "@semantic-release/github": "8.0.7",
+    "@semantic-release/npm": "9.0.2",
+    "@semantic-release/release-notes-generator": "10.0.3",
+    "commitizen": "4.2.4",
+    "cz-conventional-changelog": "3.3.0",
+    "husky": "4.0.3",
+    "lerna": "4.0.0",
+    "minimist": "1.2.8",
+    "multi-semantic-release": "3.0.2",
+    "node-fetch": "2.6.1",
+    "semantic-release": "19.0.5",
+    "semantic-release-github-pullrequest": "1.3.0",
+    "standard-version": "9.3.1"
   },
   "husky": {
     "hooks": {

packages/lb-annotation/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@labelbee/lb-annotation",
-  "version": "1.17.1-alpha.1",
+  "version": "1.17.1",
   "description": "Annotation tool collection",
   "keywords": [
     "annotation",

packages/lb-annotation/src/core/pointCloud/cache.ts

@@ -23,10 +23,14 @@ export class PointCloudCache {
 
   private static instance: PointCloudCache;
 
+  public cache2DHighlightIndex: Map<string, number[]>;
+
   private constructor() {
     this.pcdLoader = new PCDLoader();
     this.pointsMap = new Map();
     this.colorMap = new Map();
+
+    this.cache2DHighlightIndex = new Map();
   }
 
   public static getInstance() {
@@ -84,4 +88,11 @@ export class PointCloudCache {
       );
     });
   };
+
+  /**
+   * Clear all cache2dHighlightIndex.
+   */
+  public clearCache2DHighlightIndex() {
+    this.cache2DHighlightIndex.clear();
+  }
 }

packages/lb-annotation/src/core/pointCloud/highlightWorker.js

@@ -124,7 +124,7 @@ function getNewColorByBox({ zMin, zMax, polygonPointList, attribute, x, y, z, co
 }
 
 onmessage = function onmessage(e) {
-  const { position: points, color, cuboidList, colorList } = e.data;
+  const { position: points, color, cuboidList, colorList, highlightIndex } = e.data;
   let num = 0;
 
   /**
@@ -175,14 +175,22 @@ onmessage = function onmessage(e) {
       color[i + 1] = g;
       color[i + 2] = b;
     } else {
-      // // DEFAULT COLOR RENDER
-      // Recover the originPoint
-      const index = getIndex(z);
-      const newColor = COLOR_MAP_JET[index];
-      const [r, g, b] = newColor;
-      color[i] = r / 255;
-      color[i + 1] = g / 255;
-      color[i + 2] = b / 255;
+      const pointIndex = Math.floor(i / 3);
+
+      if (highlightIndex && highlightIndex[pointIndex] === 1) {
+        color[i] = 0;
+        color[i + 1] = 0;
+        color[i + 2] = 0;
+      } else {
+        // // DEFAULT COLOR RENDER
+        // Recover the originPoint
+        const index = getIndex(z);
+        const newColor = COLOR_MAP_JET[index];
+        const [r, g, b] = newColor;
+        color[i] = r / 255;
+        color[i + 1] = g / 255;
+        color[i + 2] = b / 255;
+      }
     }
   }
 

packages/lb-annotation/src/core/pointCloud/index.ts

@@ -18,6 +18,7 @@ import {
   toolStyleConverter,
   PointCloudUtils,
   DEFAULT_SPHERE_PARAMS,
+  ICalib,
 } from '@labelbee/lb-utils';
 import { BufferAttribute, OrthographicCamera, PerspectiveCamera } from 'three';
 import HighlightWorker from 'web-worker:./highlightWorker.js';
@@ -26,10 +27,11 @@ import { isInPolygon } from '@/utils/tool/polygonTool';
 import { IPolygonPoint } from '@/types/tool/polygon';
 import uuid from '@/utils/uuid';
 import MathUtils from '@/utils/MathUtils';
+import ImgUtils from '@/utils/ImgUtils';
 import { PCDLoader } from './PCDLoader';
 import { OrbitControls } from './OrbitControls';
 import { PointCloudCache } from './cache';
-import { getCuboidFromPointCloudBox } from './matrix';
+import { getCuboidFromPointCloudBox, getHighlightIndexByPoints, mergeHighlightList } from './matrix';
 import { PointCloudSegmentOperation } from './segmentation';
 import PointCloudStore from './store';
 import PointCloudRender from './render';
@@ -115,6 +117,8 @@ export class PointCloud extends EventListener {
 
   private rangeObjectName = 'range';
 
+  private highlightGroupName = 'highlightBoxes';
+
   private cacheInstance: PointCloudCache; // PointCloud Cache Map
 
   private showDirection: boolean = true; // Whether to display the direction of box
@@ -238,6 +242,10 @@ export class PointCloud extends EventListener {
     return this.segmentOperation?.currentToolName;
   }
 
+  public get pointCloudObject() {
+    return this.scene.getObjectByName(this.pointCloudObjectName) as THREE.Points;
+  }
+
   public initMsg() {
     if (!this.segmentOperation) {
       return;
@@ -943,6 +951,11 @@ export class PointCloud extends EventListener {
   public loadPCDFile = async (src: string | undefined = this.currentPCDSrc, radius?: number) => {
     if (!src) return;
     this.clearPointCloud();
+    /**
+     * Clear Img Cache.
+     */
+    this.cacheInstance.clearCache2DHighlightIndex();
+
     this.currentPCDSrc = src;
 
     /**
@@ -962,12 +975,47 @@ export class PointCloud extends EventListener {
     this.renderPointCloud(newPoints, radius);
   };
 
+  /**
+   * Highlight PointCloud by MappingImgList.
+   * @param param0
+   * @returns
+   */
+  public getHighlightIndexByMappingImgList = async ({
+    mappingImgList,
+    points,
+  }: {
+    mappingImgList: Array<{ url: string; calib: ICalib }>;
+    points: ArrayLike<number>;
+  }) => {
+    /**
+     * The img is loaded, so it can use cache in browser.
+     */
+    const imgNodeList = await Promise.all(mappingImgList.map((v) => ImgUtils.load(v.url)));
+    const highlightIndexList = mappingImgList.map((v, i) => {
+      if (this.cacheInstance.cache2DHighlightIndex.has(v.url)) {
+        return this.cacheInstance.cache2DHighlightIndex.get(v.url) ?? [];
+      }
+      const h = getHighlightIndexByPoints({
+        points,
+        calib: v.calib,
+        width: imgNodeList[i].width,
+        height: imgNodeList[i].height,
+      });
+
+      // Cache highlightIndex.
+      this.cacheInstance.cache2DHighlightIndex.set(v.url, h);
+      return h;
+    });
+    const mergeList = mergeHighlightList(highlightIndexList);
+    return mergeList;
+  };
+
   /**
    * It needs to be updated after load PointCloud's data.
    * @param boxParams
    * @returns
    */
-  public highlightOriginPointCloud(pointCloudBoxList?: IPointCloudBox[]) {
+  public highlightOriginPointCloud(pointCloudBoxList?: IPointCloudBox[], highlightIndex: number[] = []) {
     const oldPointCloud = this.scene.getObjectByName(this.pointCloudObjectName) as THREE.Points;
     if (!oldPointCloud) {
       return;
@@ -985,6 +1033,7 @@ export class PointCloud extends EventListener {
           position: oldPointCloud.geometry.attributes.position.array,
           color: oldPointCloud.geometry.attributes.dimensions.array,
           colorList,
+          highlightIndex,
         };
 
         highlightWorker.postMessage(params);
@@ -1025,6 +1074,74 @@ export class PointCloud extends EventListener {
     });
   }
 
+  /**
+   * Clean all highlightBox
+   */
+  public clearHighlightBoxes() {
+    this.removeObjectByName(this.highlightGroupName);
+  }
+
+  public clearHighlightBoxesAndRender() {
+    this.clearHighlightBoxes();
+    this.render();
+  }
+
+  public highlightBoxes(boxes: IPointCloudBox[]) {
+    const group = new THREE.Group();
+
+    // 1. Highlight all box.
+    boxes.forEach((box) => {
+      const {
+        center: { x, y, z },
+        width,
+        height,
+        depth,
+        rotation,
+      } = box;
+      const { fill } = toolStyleConverter.getColorFromConfig(
+        { attribute: box.attribute },
+        { ...this.config, attributeConfigurable: true },
+        {},
+      );
+
+      // 1-1. Add Transparent Box.
+      const geometry = new THREE.BoxGeometry(width, height, depth);
+      const material = new THREE.MeshBasicMaterial({
+        color: fill,
+        transparent: true,
+        opacity: 0.2,
+        depthTest: false,
+      });
+      geometry.rotateZ(rotation);
+      geometry.translate(x, y, z);
+      const mesh = new THREE.Mesh(geometry, material);
+      group.add(mesh);
+
+      // 1-2. Add plane for direction.
+      const planeGeo = new THREE.PlaneGeometry(depth, height);
+      planeGeo.rotateY(Math.PI / 2);
+      planeGeo.rotateZ(rotation);
+      // The plane center Offset
+      const vector = new THREE.Vector3(width / 2, 0, 0);
+      const rM = new THREE.Matrix4().makeRotationY(Math.PI / 2).makeRotationZ(rotation);
+      vector.applyMatrix4(rM);
+      planeGeo.translate(x + vector.x, y + vector.y, z + vector.z);
+      const plainMaterial = new THREE.MeshBasicMaterial({
+        color: fill,
+        side: THREE.DoubleSide,
+        transparent: true,
+        opacity: 0.8,
+        depthTest: false,
+      });
+      const plainMesh = new THREE.Mesh(planeGeo, plainMaterial);
+      group.add(plainMesh);
+    });
+
+    group.name = this.highlightGroupName;
+    this.scene.add(group);
+    this.render();
+  }
+
   public updateColor(color: any[]) {
     const oldPointCloud = this.scene.getObjectByName(this.pointCloudObjectName) as THREE.Points;
     if (oldPointCloud) {