Implement preliminary support for instanced stereo (#6967)

NEW_RELEASE_NOTES.md

@@ -7,3 +7,5 @@ for next branch cut* header.
 appropriate header in [RELEASE_NOTES.md](./RELEASE_NOTES.md).
 
 ## Release notes for next branch cut
+
+- engine: add preliminary support for instanced stereoscopic rendering [⚠️ **Recompile materials**]

docs/Materials.md.html

@@ -1139,7 +1139,8 @@
 :    array of `string`
 
 Value
-:     Each entry must be any of `dynamicLighting`, `directionalLighting`, `shadowReceiver`,`skinning` or `ssr`.
+:     Each entry must be any of `dynamicLighting`, `directionalLighting`, `shadowReceiver`,
+      `skinning`, `ssr`, or `stereo`.
 
 Description
 :     Used to specify a list of shader variants that the application guarantees will never be
@@ -1158,6 +1159,7 @@
 - `fog`, used when global fog is applied to the scene
 - `vsm`, used when VSM shadows are enabled and the object is a shadow receiver
 - `ssr`, used when screen-space reflections are enabled in the View
+- `stereo`, used when stereoscopic rendering is enabled in the View
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ JSON
 material {

filament/include/filament/Camera.h

@@ -34,10 +34,13 @@ class Entity;
 namespace filament {
 
 /**
- * Camera represents the eye through which the scene is viewed.
+ * Camera represents the eye(s) through which the scene is viewed.
  *
  * A Camera has a position and orientation and controls the projection and exposure parameters.
  *
+ * For stereoscopic rendering, a Camera maintains two separate "eyes": Eye 0 and Eye 1. These are
+ * arbitrary and don't necessarily need to correspond to "left" and "right".
+ *
  * Creation and destruction
  * ========================
  *
@@ -140,6 +143,18 @@ namespace filament {
  * intensity and the Camera exposure interact to produce the final scene's brightness.
  *
  *
+ * Stereoscopic rendering
+ * ======================
+ *
+ * The Camera's transform (as set by setModelMatrix or via TransformManager) defines a "head" space,
+ * which typically corresponds to the location of the viewer's head. Each eye's transform is set
+ * relative to this head space by setEyeModelMatrix.
+ *
+ * Each eye also maintains its own projection matrix. These can be set with setCustomEyeProjection.
+ * Care must be taken to correctly set the projectionForCulling matrix, as well as its corresponding
+ * near and far values. The projectionForCulling matrix must define a frustum (in head space) that
+ * bounds the frustums of both eyes. Alternatively, culling may be disabled with
+ * View::setFrustumCullingEnabled.
  *
  * \see Frustum, View
  */
@@ -234,6 +249,24 @@ class UTILS_PUBLIC Camera : public FilamentAPI {
      */
     void setCustomProjection(math::mat4 const& projection, double near, double far) noexcept;
 
+    /** Sets a custom projection matrix for each eye.
+     *
+     * The projectionForCulling, near, and far parameters establish a "culling frustum" which must
+     * encompass anything either eye can see.
+     *
+     * @param projection an array of projection matrices, only the first
+     *                   CONFIG_STEREOSCOPIC_EYES (2) are read
+     * @param count size of the projection matrix array to set, must be
+     *              >= CONFIG_STEREOSCOPIC_EYES (2)
+     * @param projectionForCulling custom projection matrix for culling, must encompass both eyes
+     * @param near distance in world units from the camera to the culling near plane. \p near > 0.
+     * @param far distance in world units from the camera to the culling far plane. \p far > \p
+     * near.
+     * @see setCustomProjection
+     */
+    void setCustomEyeProjection(math::mat4 const* projection, size_t count,
+            math::mat4 const& projectionForCulling, double near, double far);
+
     /** Sets the projection matrix.
      *
      * The projection matrices must be of one of the following form:
@@ -309,11 +342,14 @@ class UTILS_PUBLIC Camera : public FilamentAPI {
      * The projection matrix used for rendering always has its far plane set to infinity. This
      * is why it may differ from the matrix set through setProjection() or setLensProjection().
      *
+     * @param eyeId the index of the eye to return the projection matrix for, must be <
+     *              CONFIG_STEREOSCOPIC_EYES (2)
      * @return The projection matrix used for rendering
      *
-     * @see setProjection, setLensProjection, setCustomProjection, getCullingProjectionMatrix
+     * @see setProjection, setLensProjection, setCustomProjection, getCullingProjectionMatrix,
+     * setCustomEyeProjection
      */
-    math::mat4 getProjectionMatrix() const noexcept;
+    math::mat4 getProjectionMatrix(uint8_t eyeId = 0) const;
 
 
     /** Returns the projection matrix used for culling (far plane is finite).
@@ -350,6 +386,26 @@ class UTILS_PUBLIC Camera : public FilamentAPI {
     void setModelMatrix(const math::mat4& model) noexcept;
     void setModelMatrix(const math::mat4f& model) noexcept; //!< @overload
 
+    /** Set the position of an eye relative to this Camera (head).
+     *
+     * By default, both eyes' model matrices are identity matrices.
+     *
+     * For example, to position Eye 0 3cm leftwards and Eye 1 3cm rightwards:
+     * ~~~~~~~~~~~{.cpp}
+     * const mat4 leftEye  = mat4::translation(double3{-0.03, 0.0, 0.0});
+     * const mat4 rightEye = mat4::translation(double3{ 0.03, 0.0, 0.0});
+     * camera.setEyeModelMatrix(0, leftEye);
+     * camera.setEyeModelMatrix(1, rightEye);
+     * ~~~~~~~~~~~
+     *
+     * This method is not intended to be called every frame. Instead, to update the position of the
+     * head, use Camera::setModelMatrix.
+     *
+     * @param eyeId the index of the eye to set, must be < CONFIG_STEREOSCOPIC_EYES (2)
+     * @param model the model matrix for an individual eye
+     */
+    void setEyeModelMatrix(uint8_t eyeId, math::mat4 const& model);
+
     /** Sets the camera's model matrix
      *
      * @param eye       The position of the camera in world space.
@@ -448,7 +504,9 @@ class UTILS_PUBLIC Camera : public FilamentAPI {
     //! returns this camera's sensitivity in ISO
     float getSensitivity() const noexcept;
 
-    //! returns the focal length in meters [m] for a 35mm camera
+    /** Returns the focal length in meters [m] for a 35mm camera.
+     * Eye 0's projection matrix is used to compute the focal length.
+     */
     double getFocalLength() const noexcept;
 
     /**

filament/include/filament/Material.h

@@ -180,6 +180,11 @@ class UTILS_PUBLIC Material : public FilamentAPI {
      * is guaranteed to be invalid (either because it's been destroyed by the user already, or,
      * because it's been cleaned-up by the Engine).
      *
+     * UserVariantFilterMask::ALL should be used with caution. Only variants that an application
+     * needs should be included in the variants argument. For example, the STE variant is only used
+     * for stereoscopic rendering. If an application is not planning to render in stereo, this bit
+     * should be turned off to avoid unnecessary material compilations.
+     *
      * @param priority      Which priority queue to use, LOW or HIGH.
      * @param variants      Variants to include to the compile command.
      * @param handler       Handler to dispatch the callback or nullptr for the default handler

filament/include/filament/Options.h

@@ -541,6 +541,13 @@ struct SoftShadowOptions {
     float penumbraRatioScale = 1.0f;
 };
 
+/**
+ * Options for stereoscopic (multi-eye) rendering.
+ */
+struct StereoscopicOptions {
+    bool enabled = false;
+};
+
 } // namespace filament
 
 #endif //TNT_FILAMENT_OPTIONS_H

filament/include/filament/View.h

@@ -85,6 +85,7 @@ class UTILS_PUBLIC View : public FilamentAPI {
     using SoftShadowOptions = SoftShadowOptions;
     using ScreenSpaceReflectionsOptions = ScreenSpaceReflectionsOptions;
     using GuardBandOptions = GuardBandOptions;
+    using StereoscopicOptions = StereoscopicOptions;
 
     /**
      * Sets the View's name. Only useful for debugging.
@@ -676,6 +677,29 @@ class UTILS_PUBLIC View : public FilamentAPI {
      */
     bool isStencilBufferEnabled() const noexcept;
 
+    /**
+     * Sets the stereoscopic rendering options for this view.
+     *
+     * Currently, only one type of stereoscopic rendering is supported: side-by-side.
+     * Side-by-side stereo rendering splits the viewport into two halves: a left and right half.
+     * Eye 0 will render to the left half, while Eye 1 will render into the right half.
+     *
+     * Currently, the following features are not supported with stereoscopic rendering:
+     * - post-processing
+     * - shadowing
+     * - punctual lights
+     *
+     * @param options The stereoscopic options to use on this view
+     */
+    void setStereoscopicOptions(StereoscopicOptions const& options) noexcept;
+
+    /**
+     * Returns the stereoscopic options associated with this View.
+     *
+     * @return value set by setStereoscopicOptions().
+     */
+    StereoscopicOptions const& getStereoscopicOptions() const noexcept;
+
     // for debugging...
 
     //! debugging: allows to entirely disable frustum culling. (culling enabled by default).

filament/src/Camera.cpp

@@ -67,6 +67,15 @@ mat4 Camera::inverseProjection(const mat4 & p) noexcept {
     return filament::inverseProjection(p);
 }
 
+void Camera::setEyeModelMatrix(uint8_t eyeId, math::mat4 const& model) {
+    downcast(this)->setEyeModelMatrix(eyeId, model);
+}
+
+void Camera::setCustomEyeProjection(math::mat4 const* projection, size_t count,
+        math::mat4 const& projectionForCulling, double near, double far) {
+    downcast(this)->setCustomEyeProjection(projection, count, projectionForCulling, near, far);
+}
+
 void Camera::setProjection(Camera::Projection projection, double left, double right, double bottom,
         double top, double near, double far) {
     downcast(this)->setProjection(projection, left, right, bottom, top, near, far);
@@ -99,8 +108,8 @@ void Camera::setShift(double2 shift) noexcept {
     downcast(this)->setShift(shift);
 }
 
-mat4 Camera::getProjectionMatrix() const noexcept {
-    return downcast(this)->getUserProjectionMatrix();
+mat4 Camera::getProjectionMatrix(uint8_t eyeId) const {
+    return downcast(this)->getUserProjectionMatrix(eyeId);
 }
 
 mat4 Camera::getCullingProjectionMatrix() const noexcept {

filament/src/PerShadowMapUniforms.cpp

@@ -58,7 +58,7 @@ void PerShadowMapUniforms::prepareCamera(Transaction const& transaction,
     s.worldFromViewMatrix = worldFromView;    // model
     s.clipFromViewMatrix  = clipFromView;     // projection
     s.viewFromClipMatrix  = viewFromClip;     // 1/projection
-    s.clipFromWorldMatrix = clipFromWorld;    // projection * view
+    s.clipFromWorldMatrix[0] = clipFromWorld; // projection * view
     s.worldFromClipMatrix = worldFromClip;    // 1/(projection * view)
     s.userWorldFromWorldMatrix = mat4f(inverse(camera.worldOrigin));
     s.clipTransform = camera.clipTransfrom;

filament/src/PerViewUniforms.cpp

@@ -66,22 +66,29 @@ void PerViewUniforms::prepareCamera(FEngine& engine, const CameraInfo& camera) n
     mat4f const& clipFromView  = camera.projection;
 
     const mat4f viewFromClip{ inverse((mat4)camera.projection) };
-    const mat4f clipFromWorld{ highPrecisionMultiply(clipFromView, viewFromWorld) };
     const mat4f worldFromClip{ highPrecisionMultiply(worldFromView, viewFromClip) };
 
     auto& s = mUniforms.edit();
     s.viewFromWorldMatrix = viewFromWorld;    // view
     s.worldFromViewMatrix = worldFromView;    // model
     s.clipFromViewMatrix  = clipFromView;     // projection
     s.viewFromClipMatrix  = viewFromClip;     // 1/projection
-    s.clipFromWorldMatrix = clipFromWorld;    // projection * view
     s.worldFromClipMatrix = worldFromClip;    // 1/(projection * view)
     s.userWorldFromWorldMatrix = mat4f(inverse(camera.worldOrigin));
     s.clipTransform = camera.clipTransfrom;
     s.cameraFar = camera.zf;
     s.oneOverFarMinusNear = 1.0f / (camera.zf - camera.zn);
     s.nearOverFarMinusNear = camera.zn / (camera.zf - camera.zn);
 
+    mat4f const& headFromWorld = camera.view;
+    for (uint8_t i = 0; i < CONFIG_STEREOSCOPIC_EYES; i++) {
+        mat4f const& eyeFromHead = camera.eyeFromView[i];   // identity for monoscopic rendering
+        mat4f const& clipFromEye = camera.eyeProjection[i];
+        // clipFromEye * eyeFromHead * headFromWorld
+        s.clipFromWorldMatrix[i] = highPrecisionMultiply(
+                clipFromEye, highPrecisionMultiply(eyeFromHead, headFromWorld));
+    }
+
     // with a clip-space of [-w, w] ==> z' = -z
     // with a clip-space of [0,  w] ==> z' = (w - z)/2
     s.clipControl = engine.getDriverApi().getClipSpaceParams();

filament/src/RenderPass.cpp

@@ -458,6 +458,7 @@ RenderPass::Command* RenderPass::generateCommandsImpl(uint32_t extraFlags,
 
     const bool hasShadowing = renderFlags & HAS_SHADOWING;
     const bool viewInverseFrontFaces = renderFlags & HAS_INVERSE_FRONT_FACES;
+    const bool hasInstancedStereo = renderFlags & IS_STEREOSCOPIC;
 
     Command cmdColor;
 
@@ -522,6 +523,15 @@ RenderPass::Command* RenderPass::generateCommandsImpl(uint32_t extraFlags,
                 soaInstanceInfo[i].count | PrimitiveInfo::USER_INSTANCE_MASK;
         cmdColor.primitive.instanceBufferHandle = soaInstanceInfo[i].handle;
 
+        // soaInstanceInfo[i].count is the number of instances the user has requested, either for
+        // manual or hybrid instancing. Instanced stereo multiplies the number of instances by the
+        // eye count.
+        if (UTILS_UNLIKELY(hasInstancedStereo)) {
+            cmdColor.primitive.instanceCount =
+                    (soaInstanceInfo[i].count * CONFIG_STEREOSCOPIC_EYES) |
+                    PrimitiveInfo::USER_INSTANCE_MASK;
+        }
+
         // if we are already an SSR variant, the SRE bit is already set,
         // there is no harm setting it again
         static_assert(Variant::SPECIAL_SSR & Variant::SRE);
@@ -541,6 +551,12 @@ RenderPass::Command* RenderPass::generateCommandsImpl(uint32_t extraFlags,
             cmdDepth.primitive.instanceBufferHandle = soaInstanceInfo[i].handle;
             cmdDepth.primitive.materialVariant.setSkinning(hasSkinningOrMorphing);
             cmdDepth.primitive.rasterState.inverseFrontFaces = inverseFrontFaces;
+
+            if (UTILS_UNLIKELY(hasInstancedStereo)) {
+                cmdColor.primitive.instanceCount =
+                        (soaInstanceInfo[i].count * CONFIG_STEREOSCOPIC_EYES) |
+                        PrimitiveInfo::USER_INSTANCE_MASK;
+            }
         }
         if constexpr (isColorPass) {
             renderableVariant.setFog(soaVisibility[i].fog && Variant::isFogVariant(variant));

filament/src/RenderPass.h

@@ -264,6 +264,7 @@ class RenderPass {
     using RenderFlags = uint8_t;
     static constexpr RenderFlags HAS_SHADOWING           = 0x01;
     static constexpr RenderFlags HAS_INVERSE_FRONT_FACES = 0x02;
+    static constexpr RenderFlags IS_STEREOSCOPIC         = 0x04;
 
     // Arena used for commands
     using Arena = utils::Arena<

filament/src/View.cpp

@@ -283,6 +283,14 @@ bool View::isStencilBufferEnabled() const noexcept {
     return downcast(this)->isStencilBufferEnabled();
 }
 
+void View::setStereoscopicOptions(const StereoscopicOptions& options) noexcept {
+    return downcast(this)->setStereoscopicOptions(options);
+}
+
+const View::StereoscopicOptions& View::getStereoscopicOptions() const noexcept {
+    return downcast(this)->getStereoscopicOptions();
+}
+
 View::PickingQuery& View::pick(uint32_t x, uint32_t y, backend::CallbackHandler* handler,
         View::PickingQueryResultCallback callback) noexcept {
     return downcast(this)->pick(x, y, handler, callback);