From 7622d7002dc6d084d06e6413b78f7efd91adba80 Mon Sep 17 00:00:00 2001 From: lby Date: Wed, 19 Jul 2023 10:00:35 +0800 Subject: [PATCH] chore(web): add image-based lighting (#519) Co-authored-by: keiya sasaki Co-authored-by: pyshx --- web/src/beta/lib/core/Map/Layer/index.tsx | 3 +- web/src/beta/lib/core/Map/types/index.ts | 3 + .../engines/Cesium/Feature/Model/index.tsx | 102 +++++++++++++++++- .../engines/Cesium/Feature/Tileset/hooks.ts | 55 ++++++++++ .../engines/Cesium/Feature/Tileset/index.tsx | 4 +- .../lib/core/engines/Cesium/Feature/index.tsx | 25 ++++- .../lib/core/engines/Cesium/Feature/utils.tsx | 10 +- .../lib/core/mantle/evaluator/simple/index.ts | 2 +- .../beta/lib/core/mantle/types/appearance.ts | 6 ++ 9 files changed, 199 insertions(+), 11 deletions(-) diff --git a/web/src/beta/lib/core/Map/Layer/index.tsx b/web/src/beta/lib/core/Map/Layer/index.tsx index 4d1d82afcd..bbb5f53236 100644 --- a/web/src/beta/lib/core/Map/Layer/index.tsx +++ b/web/src/beta/lib/core/Map/Layer/index.tsx @@ -8,6 +8,7 @@ import type { DataType, ComputedFeature, } from "../../mantle"; +import { SceneProperty } from "../types"; import useHooks, { type Atom, type EvalFeature } from "./hooks"; @@ -27,7 +28,7 @@ export type CommonProps = { export type FeatureComponentProps = { layer: ComputedLayer; - sceneProperty?: any; + sceneProperty?: SceneProperty; onFeatureRequest?: (range: DataRange) => void; onFeatureFetch?: (features: Feature[]) => void; onComputedFeatureFetch?: (feature: Feature[], computed: ComputedFeature[]) => void; diff --git a/web/src/beta/lib/core/Map/types/index.ts b/web/src/beta/lib/core/Map/types/index.ts index fbfae0ef8c..c4f0e35d90 100644 --- a/web/src/beta/lib/core/Map/types/index.ts +++ b/web/src/beta/lib/core/Map/types/index.ts @@ -331,6 +331,9 @@ export type SceneProperty = { lightDirectionZ?: number; lightColor?: string; lightIntensity?: number; + specularEnvironmentMaps?: string; + sphericalHarmonicCoefficients?: [x: number, y: number, z: number][]; + imageBasedLightIntensity?: number; }; render?: { antialias?: "low" | "medium" | "high" | "extreme"; diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/Model/index.tsx b/web/src/beta/lib/core/engines/Cesium/Feature/Model/index.tsx index 75d1e285ec..e648862a87 100644 --- a/web/src/beta/lib/core/engines/Cesium/Feature/Model/index.tsx +++ b/web/src/beta/lib/core/engines/Cesium/Feature/Model/index.tsx @@ -1,12 +1,20 @@ -import { Cartesian3, HeadingPitchRoll, Math as CesiumMath, Transforms } from "cesium"; -import { useMemo } from "react"; -import { ModelGraphics } from "resium"; +import { + Cartesian3, + HeadingPitchRoll, + Math as CesiumMath, + Transforms, + Model as CesiumModel, + ImageBasedLighting, +} from "cesium"; +import { useEffect, useMemo, useRef } from "react"; +import { ModelGraphics, useCesium } from "resium"; import { toColor } from "@reearth/beta/utils/value"; import type { ModelAppearance } from "../../.."; import { colorBlendMode, heightReference, shadowMode } from "../../common"; import { NonPBRLightingShader } from "../../CustomShaders/NonPBRLightingShader"; +import { useSceneEvent } from "../../hooks/useSceneEvent"; import { EntityExt, extractSimpleLayerData, @@ -23,7 +31,27 @@ export type Property = ModelAppearance & { height?: number; }; -export default function Model({ id, isVisible, property, geometry, layer, feature }: Props) { +const sphericalHarmonicCoefficientsScratch = [ + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), + new Cartesian3(), +]; + +export default function Model({ + id, + isVisible, + property, + sceneProperty, + geometry, + layer, + feature, +}: Props) { const data = extractSimpleLayerData(layer); const isGltfData = data?.type === "gltf"; @@ -94,6 +122,72 @@ export default function Model({ id, isVisible, property, geometry, layer, featur [property?.near, property?.far], ); + const imageBasedLighting = useMemo(() => { + const ibl = new ImageBasedLighting(); + + if ( + !property?.specularEnvironmentMaps && + !property?.sphericalHarmonicCoefficients && + !sceneProperty?.light?.specularEnvironmentMaps && + !sceneProperty?.light?.sphericalHarmonicCoefficients + ) + return ibl; + + const specularEnvironmentMaps = + property?.specularEnvironmentMaps ?? sceneProperty?.light?.specularEnvironmentMaps; + const sphericalHarmonicCoefficients = + property?.sphericalHarmonicCoefficients ?? + sceneProperty?.light?.sphericalHarmonicCoefficients; + const imageBasedLightIntensity = + property?.imageBasedLightIntensity ?? sceneProperty?.light?.imageBasedLightIntensity; + + if (specularEnvironmentMaps) { + ibl.specularEnvironmentMaps = specularEnvironmentMaps; + } + if (sphericalHarmonicCoefficients) { + ibl.sphericalHarmonicCoefficients = sphericalHarmonicCoefficients?.map((cartesian, index) => + Cartesian3.multiplyByScalar( + new Cartesian3(...cartesian), + imageBasedLightIntensity ?? 1.0, + sphericalHarmonicCoefficientsScratch[index], + ), + ); + } + return ibl; + }, [ + property?.specularEnvironmentMaps, + property?.sphericalHarmonicCoefficients, + property?.imageBasedLightIntensity, + sceneProperty?.light?.specularEnvironmentMaps, + sceneProperty?.light?.sphericalHarmonicCoefficients, + sceneProperty?.light?.imageBasedLightIntensity, + ]); + + const { viewer } = useCesium(); + const shouldUpdateAfterLoaded = useRef(false); + useSceneEvent("postRender", () => { + const primitives = viewer?.scene.primitives; + const length = primitives?.length ?? 0; + + if (!shouldUpdateAfterLoaded.current || !imageBasedLighting) { + return; + } + + for (let i = 0; i < length; i++) { + const prim = primitives?.get(i); + if (prim instanceof CesiumModel && prim.id && prim.id.id === id) { + shouldUpdateAfterLoaded.current = false; + prim.imageBasedLighting = imageBasedLighting; + } + } + }); + + useEffect(() => { + if (imageBasedLighting) { + shouldUpdateAfterLoaded.current = true; + } + }, [imageBasedLighting]); + // if data type is gltf, layer should be rendered. Otherwise only features should be rendererd. return (isGltfData ? feature : !feature) || !isVisible || !show || !actualUrl ? null : ( { return useMemo(() => { const data = extractSimpleLayerData(layer); @@ -297,6 +310,7 @@ export const useHooks = ({ boxId, isVisible, property, + sceneProperty, layer, feature, meta, @@ -516,11 +530,52 @@ export const useHooks = ({ : null; }, [isVisible, tileset, url, type, meta]); + const imageBasedLighting = useMemo(() => { + if ( + !property?.specularEnvironmentMaps && + !property?.sphericalHarmonicCoefficients && + !sceneProperty?.light?.specularEnvironmentMaps && + !sceneProperty?.light?.sphericalHarmonicCoefficients + ) + return; + + const ibl = new ImageBasedLighting(); + const specularEnvironmentMaps = + property?.specularEnvironmentMaps ?? sceneProperty?.light?.specularEnvironmentMaps; + const sphericalHarmonicCoefficients = + property?.sphericalHarmonicCoefficients ?? + sceneProperty?.light?.sphericalHarmonicCoefficients; + const imageBasedLightIntensity = + property?.imageBasedLightIntensity ?? sceneProperty?.light?.imageBasedLightIntensity; + + if (specularEnvironmentMaps) { + ibl.specularEnvironmentMaps = specularEnvironmentMaps; + } + if (sphericalHarmonicCoefficients) { + ibl.sphericalHarmonicCoefficients = sphericalHarmonicCoefficients?.map((cartesian, index) => + Cartesian3.multiplyByScalar( + new Cartesian3(...cartesian), + imageBasedLightIntensity ?? 1.0, + sphericalHarmonicCoefficientsScratch[index], + ), + ); + } + return ibl; + }, [ + property?.specularEnvironmentMaps, + property?.sphericalHarmonicCoefficients, + property?.imageBasedLightIntensity, + sceneProperty?.light?.specularEnvironmentMaps, + sceneProperty?.light?.sphericalHarmonicCoefficients, + sceneProperty?.light?.imageBasedLightIntensity, + ]); + return { tilesetUrl, ref, style, clippingPlanes, builtinBoxProps, + imageBasedLighting, }; }; diff --git a/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/index.tsx b/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/index.tsx index bab9a60519..5e8312d051 100644 --- a/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/index.tsx +++ b/web/src/beta/lib/core/engines/Cesium/Feature/Tileset/index.tsx @@ -26,13 +26,14 @@ function Tileset({ }: Props): JSX.Element | null { const { shadows, colorBlendMode, pbr } = property ?? {}; const boxId = `${layer?.id}_box`; - const { tilesetUrl, ref, style, clippingPlanes, builtinBoxProps } = useHooks({ + const { tilesetUrl, ref, style, clippingPlanes, builtinBoxProps, imageBasedLighting } = useHooks({ id, boxId, isVisible, layer, feature, property, + sceneProperty, meta, evalFeature, }); @@ -55,6 +56,7 @@ function Tileset({ shadows={shadowMode(shadows)} clippingPlanes={clippingPlanes} colorBlendMode={colorBlendModeFor3DTile(colorBlendMode)} + imageBasedLighting={imageBasedLighting} /> {builtinBoxProps && ( { + const useSceneSphericalHarmonicCoefficients = + !!props.sceneProperty?.light?.sphericalHarmonicCoefficients; + const useSceneSpecularEnvironmentMaps = !!props.sceneProperty?.light?.specularEnvironmentMaps; + const componentId = generateIDWithMD5( - `${layer.id}_${f?.id ?? ""}_${k}_${isHidden}_${data?.url ?? ""}_${ + `${layer.id}_${f?.id ?? ""}_${k}_${isHidden}_${ + data?.url ?? "" + }_${useSceneSphericalHarmonicCoefficients}_${useSceneSpecularEnvironmentMaps}_${ JSON.stringify(f?.[k]) ?? "" }`, ); @@ -161,9 +167,24 @@ export default function Feature({ const [C] = components[k] ?? []; const isVisible = layer.layer.visible !== false && !isHidden; + // NOTE: IBL for 3dtiles is not updated unless Tileset feature component is re-created. + const useSceneSphericalHarmonicCoefficients = + !!props.sceneProperty?.light?.sphericalHarmonicCoefficients; + const useSceneSpecularEnvironmentMaps = + !!props.sceneProperty?.light?.specularEnvironmentMaps; + const use3dtilesSphericalHarmonicCoefficients = + layer?.layer?.type === "simple" && + !!layer?.layer?.["3dtiles"]?.sphericalHarmonicCoefficients; + const use3dtilesSpecularEnvironmentMaps = + layer?.layer?.type === "simple" && !!layer?.layer?.["3dtiles"]?.specularEnvironmentMaps; + // "noFeature" component should be recreated when the following value is changed. // data.url, isVisible - const key = generateIDWithMD5(`${layer?.id || ""}_${k}_${data?.url}_${isVisible}`); + const key = generateIDWithMD5( + `${layer?.id || ""}_${k}_${ + data?.url + }_${isVisible}_${useSceneSphericalHarmonicCoefficients}_${useSceneSpecularEnvironmentMaps}_${use3dtilesSphericalHarmonicCoefficients}}_${use3dtilesSpecularEnvironmentMaps}`, + ); return ( = { @@ -39,7 +45,7 @@ export type FeatureProps

= { layer?: ComputedLayer; feature?: ComputedFeature; geometry?: Geometry; - sceneProperty?: any; + sceneProperty?: SceneProperty; } & Omit; export type FeatureComponent = ComponentType; diff --git a/web/src/beta/lib/core/mantle/evaluator/simple/index.ts b/web/src/beta/lib/core/mantle/evaluator/simple/index.ts index 494da1cda2..6598dd3cd4 100644 --- a/web/src/beta/lib/core/mantle/evaluator/simple/index.ts +++ b/web/src/beta/lib/core/mantle/evaluator/simple/index.ts @@ -120,7 +120,7 @@ function hasExpression(e: any): e is ExpressionContainer { // eslint-disable-next-line @typescript-eslint/no-explicit-any function hasNonExpressionObject(v: any): boolean { - return typeof v === "object" && v && !("expression" in v); + return typeof v === "object" && v && !("expression" in v) && !Array.isArray(v); } function evalExpression( diff --git a/web/src/beta/lib/core/mantle/types/appearance.ts b/web/src/beta/lib/core/mantle/types/appearance.ts index c00809af13..8cd3f03f60 100644 --- a/web/src/beta/lib/core/mantle/types/appearance.ts +++ b/web/src/beta/lib/core/mantle/types/appearance.ts @@ -132,6 +132,9 @@ export type ModelAppearance = { near?: number; far?: number; pbr?: boolean; + specularEnvironmentMaps?: string; + sphericalHarmonicCoefficients?: [x: number, y: number, z: number][]; + imageBasedLightIntensity?: number; }; export type Cesium3DTilesAppearance = { @@ -147,6 +150,9 @@ export type Cesium3DTilesAppearance = { pointSize?: number; meta?: unknown; pbr?: boolean; + specularEnvironmentMaps?: string; + sphericalHarmonicCoefficients?: [x: number, y: number, z: number][]; + imageBasedLightIntensity?: number; }; export type LegacyPhotooverlayAppearance = {