chore(web): add image-based lighting (#519)

Co-authored-by: keiya sasaki <[email protected]> Co-authored-by: pyshx <[email protected]>
reearth · Jul 19, 2023 · 7622d70 · 7622d70
1 parent 9473a5a
commit 7622d70
Show file tree

Hide file tree

Showing 9 changed files with 199 additions and 11 deletions.
diff --git a/web/src/beta/lib/core/Map/Layer/index.tsx b/web/src/beta/lib/core/Map/Layer/index.tsx
@@ -8,6 +8,7 @@ import type {
   DataType,
   ComputedFeature,
 } from "../../mantle";
+import { SceneProperty } from "../types";
 
 import useHooks, { type Atom, type EvalFeature } from "./hooks";
 
@@ -27,7 +28,7 @@ export type CommonProps = {
 
 export type FeatureComponentProps = {
   layer: ComputedLayer;
-  sceneProperty?: any;
+  sceneProperty?: SceneProperty;
   onFeatureRequest?: (range: DataRange) => void;
   onFeatureFetch?: (features: Feature[]) => void;
   onComputedFeatureFetch?: (feature: Feature[], computed: ComputedFeature[]) => void;

diff --git a/web/src/beta/lib/core/Map/types/index.ts b/web/src/beta/lib/core/Map/types/index.ts
@@ -331,6 +331,9 @@ export type SceneProperty = {
     lightDirectionZ?: number;
     lightColor?: string;
     lightIntensity?: number;
+    specularEnvironmentMaps?: string;
+    sphericalHarmonicCoefficients?: [x: number, y: number, z: number][];
+    imageBasedLightIntensity?: number;
   };
   render?: {
     antialias?: "low" | "medium" | "high" | "extreme";

diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/Model/index.tsx b/web/src/beta/lib/core/engines/Cesium/Feature/Model/index.tsx
@@ -1,12 +1,20 @@
-import { Cartesian3, HeadingPitchRoll, Math as CesiumMath, Transforms } from "cesium";
-import { useMemo } from "react";
-import { ModelGraphics } from "resium";
+import {
+  Cartesian3,
+  HeadingPitchRoll,
+  Math as CesiumMath,
+  Transforms,
+  Model as CesiumModel,
+  ImageBasedLighting,
+} from "cesium";
+import { useEffect, useMemo, useRef } from "react";
+import { ModelGraphics, useCesium } from "resium";
 
 import { toColor } from "@reearth/beta/utils/value";
 
 import type { ModelAppearance } from "../../..";
 import { colorBlendMode, heightReference, shadowMode } from "../../common";
 import { NonPBRLightingShader } from "../../CustomShaders/NonPBRLightingShader";
+import { useSceneEvent } from "../../hooks/useSceneEvent";
 import {
   EntityExt,
   extractSimpleLayerData,
@@ -23,7 +31,27 @@ export type Property = ModelAppearance & {
   height?: number;
 };
 
-export default function Model({ id, isVisible, property, geometry, layer, feature }: Props) {
+const sphericalHarmonicCoefficientsScratch = [
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+];
+
+export default function Model({
+  id,
+  isVisible,
+  property,
+  sceneProperty,
+  geometry,
+  layer,
+  feature,
+}: Props) {
   const data = extractSimpleLayerData(layer);
   const isGltfData = data?.type === "gltf";
 
@@ -94,6 +122,72 @@ export default function Model({ id, isVisible, property, geometry, layer, featur
     [property?.near, property?.far],
   );
 
+  const imageBasedLighting = useMemo(() => {
+    const ibl = new ImageBasedLighting();
+
+    if (
+      !property?.specularEnvironmentMaps &&
+      !property?.sphericalHarmonicCoefficients &&
+      !sceneProperty?.light?.specularEnvironmentMaps &&
+      !sceneProperty?.light?.sphericalHarmonicCoefficients
+    )
+      return ibl;
+
+    const specularEnvironmentMaps =
+      property?.specularEnvironmentMaps ?? sceneProperty?.light?.specularEnvironmentMaps;
+    const sphericalHarmonicCoefficients =
+      property?.sphericalHarmonicCoefficients ??
+      sceneProperty?.light?.sphericalHarmonicCoefficients;
+    const imageBasedLightIntensity =
+      property?.imageBasedLightIntensity ?? sceneProperty?.light?.imageBasedLightIntensity;
+
+    if (specularEnvironmentMaps) {
+      ibl.specularEnvironmentMaps = specularEnvironmentMaps;
+    }
+    if (sphericalHarmonicCoefficients) {
+      ibl.sphericalHarmonicCoefficients = sphericalHarmonicCoefficients?.map((cartesian, index) =>
+        Cartesian3.multiplyByScalar(
+          new Cartesian3(...cartesian),
+          imageBasedLightIntensity ?? 1.0,
+          sphericalHarmonicCoefficientsScratch[index],
+        ),
+      );
+    }
+    return ibl;
+  }, [
+    property?.specularEnvironmentMaps,
+    property?.sphericalHarmonicCoefficients,
+    property?.imageBasedLightIntensity,
+    sceneProperty?.light?.specularEnvironmentMaps,
+    sceneProperty?.light?.sphericalHarmonicCoefficients,
+    sceneProperty?.light?.imageBasedLightIntensity,
+  ]);
+
+  const { viewer } = useCesium();
+  const shouldUpdateAfterLoaded = useRef(false);
+  useSceneEvent("postRender", () => {
+    const primitives = viewer?.scene.primitives;
+    const length = primitives?.length ?? 0;
+
+    if (!shouldUpdateAfterLoaded.current || !imageBasedLighting) {
+      return;
+    }
+
+    for (let i = 0; i < length; i++) {
+      const prim = primitives?.get(i);
+      if (prim instanceof CesiumModel && prim.id && prim.id.id === id) {
+        shouldUpdateAfterLoaded.current = false;
+        prim.imageBasedLighting = imageBasedLighting;
+      }
+    }
+  });
+
+  useEffect(() => {
+    if (imageBasedLighting) {
+      shouldUpdateAfterLoaded.current = true;
+    }
+  }, [imageBasedLighting]);
+
   // if data type is gltf, layer should be rendered. Otherwise only features should be rendererd.
   return (isGltfData ? feature : !feature) || !isVisible || !show || !actualUrl ? null : (
     <EntityExt

diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/hooks.ts b/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/hooks.ts
@@ -15,6 +15,7 @@ import {
   Cesium3DTileFeature,
   Model,
   Cesium3DTilePointFeature,
+  ImageBasedLighting,
 } from "cesium";
 import { isEqual, pick } from "lodash-es";
 import { MutableRefObject, useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -44,6 +45,18 @@ import { useClippingBox } from "./useClippingBox";
 
 import { Property } from ".";
 
+const sphericalHarmonicCoefficientsScratch = [
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+  new Cartesian3(),
+];
+
 const useData = (layer: ComputedLayer | undefined) => {
   return useMemo(() => {
     const data = extractSimpleLayerData(layer);
@@ -297,6 +310,7 @@ export const useHooks = ({
   boxId,
   isVisible,
   property,
+  sceneProperty,
   layer,
   feature,
   meta,
@@ -516,11 +530,52 @@ export const useHooks = ({
       : null;
   }, [isVisible, tileset, url, type, meta]);
 
+  const imageBasedLighting = useMemo(() => {
+    if (
+      !property?.specularEnvironmentMaps &&
+      !property?.sphericalHarmonicCoefficients &&
+      !sceneProperty?.light?.specularEnvironmentMaps &&
+      !sceneProperty?.light?.sphericalHarmonicCoefficients
+    )
+      return;
+
+    const ibl = new ImageBasedLighting();
+    const specularEnvironmentMaps =
+      property?.specularEnvironmentMaps ?? sceneProperty?.light?.specularEnvironmentMaps;
+    const sphericalHarmonicCoefficients =
+      property?.sphericalHarmonicCoefficients ??
+      sceneProperty?.light?.sphericalHarmonicCoefficients;
+    const imageBasedLightIntensity =
+      property?.imageBasedLightIntensity ?? sceneProperty?.light?.imageBasedLightIntensity;
+
+    if (specularEnvironmentMaps) {
+      ibl.specularEnvironmentMaps = specularEnvironmentMaps;
+    }
+    if (sphericalHarmonicCoefficients) {
+      ibl.sphericalHarmonicCoefficients = sphericalHarmonicCoefficients?.map((cartesian, index) =>
+        Cartesian3.multiplyByScalar(
+          new Cartesian3(...cartesian),
+          imageBasedLightIntensity ?? 1.0,
+          sphericalHarmonicCoefficientsScratch[index],
+        ),
+      );
+    }
+    return ibl;
+  }, [
+    property?.specularEnvironmentMaps,
+    property?.sphericalHarmonicCoefficients,
+    property?.imageBasedLightIntensity,
+    sceneProperty?.light?.specularEnvironmentMaps,
+    sceneProperty?.light?.sphericalHarmonicCoefficients,
+    sceneProperty?.light?.imageBasedLightIntensity,
+  ]);
+
   return {
     tilesetUrl,
     ref,
     style,
     clippingPlanes,
     builtinBoxProps,
+    imageBasedLighting,
   };
 };
diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/index.tsx b/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/index.tsx
@@ -26,13 +26,14 @@ function Tileset({
 }: Props): JSX.Element | null {
   const { shadows, colorBlendMode, pbr } = property ?? {};
   const boxId = `${layer?.id}_box`;
-  const { tilesetUrl, ref, style, clippingPlanes, builtinBoxProps } = useHooks({
+  const { tilesetUrl, ref, style, clippingPlanes, builtinBoxProps, imageBasedLighting } = useHooks({
     id,
     boxId,
     isVisible,
     layer,
     feature,
     property,
+    sceneProperty,
     meta,
     evalFeature,
   });
@@ -55,6 +56,7 @@ function Tileset({
         shadows={shadowMode(shadows)}
         clippingPlanes={clippingPlanes}
         colorBlendMode={colorBlendModeFor3DTile(colorBlendMode)}
+        imageBasedLighting={imageBasedLighting}
       />
       {builtinBoxProps && (
         <Box

diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/index.tsx b/web/src/beta/lib/core/engines/Cesium/Feature/index.tsx
@@ -93,8 +93,14 @@ export default function Feature({
   const cacheable = !data?.updateInterval;
 
   const renderComponent = (k: keyof AppearanceTypes, f?: ComputedFeature): JSX.Element | null => {
+    const useSceneSphericalHarmonicCoefficients =
+      !!props.sceneProperty?.light?.sphericalHarmonicCoefficients;
+    const useSceneSpecularEnvironmentMaps = !!props.sceneProperty?.light?.specularEnvironmentMaps;
+
     const componentId = generateIDWithMD5(
-      `${layer.id}_${f?.id ?? ""}_${k}_${isHidden}_${data?.url ?? ""}_${
+      `${layer.id}_${f?.id ?? ""}_${k}_${isHidden}_${
+        data?.url ?? ""
+      }_${useSceneSphericalHarmonicCoefficients}_${useSceneSpecularEnvironmentMaps}_${
         JSON.stringify(f?.[k]) ?? ""
       }`,
     );
@@ -161,9 +167,24 @@ export default function Feature({
           const [C] = components[k] ?? [];
           const isVisible = layer.layer.visible !== false && !isHidden;
 
+          // NOTE: IBL for 3dtiles is not updated unless Tileset feature component is re-created.
+          const useSceneSphericalHarmonicCoefficients =
+            !!props.sceneProperty?.light?.sphericalHarmonicCoefficients;
+          const useSceneSpecularEnvironmentMaps =
+            !!props.sceneProperty?.light?.specularEnvironmentMaps;
+          const use3dtilesSphericalHarmonicCoefficients =
+            layer?.layer?.type === "simple" &&
+            !!layer?.layer?.["3dtiles"]?.sphericalHarmonicCoefficients;
+          const use3dtilesSpecularEnvironmentMaps =
+            layer?.layer?.type === "simple" && !!layer?.layer?.["3dtiles"]?.specularEnvironmentMaps;
+
           // "noFeature" component should be recreated when the following value is changed.
           // data.url, isVisible
-          const key = generateIDWithMD5(`${layer?.id || ""}_${k}_${data?.url}_${isVisible}`);
+          const key = generateIDWithMD5(
+            `${layer?.id || ""}_${k}_${
+              data?.url
+            }_${isVisible}_${useSceneSphericalHarmonicCoefficients}_${useSceneSpecularEnvironmentMaps}_${use3dtilesSphericalHarmonicCoefficients}}_${use3dtilesSpecularEnvironmentMaps}`,
+          );
 
           return (
             <C

diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/utils.tsx b/web/src/beta/lib/core/engines/Cesium/Feature/utils.tsx
@@ -29,7 +29,13 @@ import { type CesiumComponentRef, Entity } from "resium";
 
 import { Data, Layer, LayerSimple, TimeInterval } from "@reearth/beta/lib/core/mantle";
 
-import type { ComputedFeature, ComputedLayer, FeatureComponentProps, Geometry } from "../..";
+import type {
+  ComputedFeature,
+  ComputedLayer,
+  FeatureComponentProps,
+  Geometry,
+  SceneProperty,
+} from "../..";
 import type { InternalCesium3DTileFeature } from "../types";
 
 export type FeatureProps<P = any> = {
@@ -39,7 +45,7 @@ export type FeatureProps<P = any> = {
   layer?: ComputedLayer;
   feature?: ComputedFeature;
   geometry?: Geometry;
-  sceneProperty?: any;
+  sceneProperty?: SceneProperty;
 } & Omit<FeatureComponentProps, "layer">;
 
 export type FeatureComponent = ComponentType<FeatureProps>;

diff --git a/web/src/beta/lib/core/mantle/evaluator/simple/index.ts b/web/src/beta/lib/core/mantle/evaluator/simple/index.ts
@@ -120,7 +120,7 @@ function hasExpression(e: any): e is ExpressionContainer {
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function hasNonExpressionObject(v: any): boolean {
-  return typeof v === "object" && v && !("expression" in v);
+  return typeof v === "object" && v && !("expression" in v) && !Array.isArray(v);
 }
 
 function evalExpression(

diff --git a/web/src/beta/lib/core/mantle/types/appearance.ts b/web/src/beta/lib/core/mantle/types/appearance.ts
@@ -132,6 +132,9 @@ export type ModelAppearance = {
   near?: number;
   far?: number;
   pbr?: boolean;
+  specularEnvironmentMaps?: string;
+  sphericalHarmonicCoefficients?: [x: number, y: number, z: number][];
+  imageBasedLightIntensity?: number;
 };
 
 export type Cesium3DTilesAppearance = {
@@ -147,6 +150,9 @@ export type Cesium3DTilesAppearance = {
   pointSize?: number;
   meta?: unknown;
   pbr?: boolean;
+  specularEnvironmentMaps?: string;
+  sphericalHarmonicCoefficients?: [x: number, y: number, z: number][];
+  imageBasedLightIntensity?: number;
 };
 
 export type LegacyPhotooverlayAppearance = {