Archiver: optimize GLSL shaders (close #469)

Graphics/Archiver/CMakeLists.txt

@@ -126,7 +126,12 @@ if(VULKAN_SUPPORTED)
 endif()
 
 if(GL_SUPPORTED OR GLES_SUPPORTED)
-    target_link_libraries(Diligent-Archiver-static PRIVATE Diligent-GraphicsEngineOpenGL-static)
+    target_link_libraries(Diligent-Archiver-static
+    PRIVATE
+        Diligent-GraphicsEngineOpenGL-static
+        spirv-cross-core
+        spirv-cross-glsl
+    )
     target_include_directories(Diligent-Archiver-static PRIVATE ../GraphicsEngineOpenGL/include)
 
     if(PLATFORM_WIN32)

Graphics/Archiver/include/SerializationDeviceImpl.hpp

@@ -145,7 +145,8 @@ class SerializationDeviceImpl final : public RenderDeviceBase<SerializationEngin
 
     struct GLProperties
     {
-        bool ValidateShaders = false;
+        bool OptimizeShaders = false;
+        bool ZeroToOneClipZ  = false;
     };
 
     struct VkProperties

Graphics/Archiver/interface/ArchiverFactory.h

@@ -98,20 +98,33 @@ typedef struct SerializationDeviceD3D12Info SerializationDeviceD3D12Info;
 /// Serialization device attributes for OpenGL backend
 struct SerializationDeviceGLInfo
 {
-    /// Whether to validate OpenGL shaders.
+    /// Whether to optimize OpenGL shaders.
+
+    /// \remarks    In OpenGL backend, shaders are stored as source code in the archive.
+    ///             The source code can be rather large since all included files are inlined,
+    ///             helper shader definitions are added, etc. Compiling such shaders may take
+    ///             a significant amount of time, in particular on mobile devices and WebGL.
+    ///             When OptimizeShaders is set to true, the archiver will optimize the shader
+    ///             source code for run-time loading performance.
+    ///
+    ///             Technical details: the archiver will compile the shader source code to SPIRV
+    ///             with GLSLang and then translate SPIRV back to GLSL using SPIRV-Cross.
+    ///             The resulting GLSL code will be much more compact and will be stored in the
+    ///             archive instead of the original source code.
+    Bool  OptimizeShaders DEFAULT_INITIALIZER(True);
 
-    /// \remarks    In OpenGL backend, shaders are converted from HLSL to GLSL
-    ///             (if necessary) and packed into an archive as source code.
-    ///             When this flag is set to True, the archiver will compile
-    ///             the source code to validate it is correct.
-    ///             This may be time-consuming and could be disabled to speed up
-    ///             the archiving process.
-    Bool  ValidateShaders DEFAULT_INITIALIZER(True);
+    /// Whether to use zero-to-one clip-space Z range.
+    ///
+    /// \remarks    In OpenGL, the default clip-space Z range is -1 to 1.
+    ///             When this flag is set to True, the archiver will assume
+    ///             that the shaders use zero-to-one clip-space Z range.
+    Bool ZeroToOneClipZ DEFAULT_INITIALIZER(False);
 
 #if DILIGENT_CPP_INTERFACE
     bool operator==(const SerializationDeviceGLInfo& RHS) const noexcept
     {
-        return ValidateShaders == RHS.ValidateShaders;
+        return OptimizeShaders == RHS.OptimizeShaders &&
+			   ZeroToOneClipZ  == RHS.ZeroToOneClipZ;
     }
     bool operator!=(const SerializationDeviceGLInfo& RHS) const noexcept
     {

Graphics/Archiver/src/Archiver_GL.cpp

@@ -40,6 +40,7 @@
 #if !DILIGENT_NO_GLSLANG
 #    include "GLSLUtils.hpp"
 #    include "GLSLangUtils.hpp"
+#    include "spirv_glsl.hpp"
 #endif
 
 namespace Diligent
@@ -59,15 +60,28 @@ namespace
 
 struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
 {
-    const String           UnrolledSource;
+    String                 UnrolledSource;
     RefCntAutoPtr<IShader> pShaderGL;
-
-    CompiledShaderGL(IReferenceCounters*             pRefCounters,
-                     const ShaderCreateInfo&         ShaderCI,
-                     const ShaderGLImpl::CreateInfo& GLShaderCI,
-                     IRenderDevice*                  pRenderDeviceGL) :
-        UnrolledSource{UnrollSource(ShaderCI)}
+    bool                   IsOptimized = false;
+
+    CompiledShaderGL(IReferenceCounters*                          pRefCounters,
+                     const ShaderCreateInfo&                      ShaderCI,
+                     const ShaderGLImpl::CreateInfo&              GLShaderCI,
+                     IRenderDevice*                               pRenderDeviceGL,
+                     RENDER_DEVICE_TYPE                           DeviceType,
+                     const SerializationDeviceImpl::GLProperties& GLProps)
     {
+        if (GLProps.OptimizeShaders)
+        {
+            UnrolledSource = TransformSource(ShaderCI, GLShaderCI, DeviceType, GLProps);
+            IsOptimized    = !UnrolledSource.empty();
+        }
+        if (UnrolledSource.empty())
+        {
+            UnrolledSource = UnrollSource(ShaderCI);
+        }
+        VERIFY_EXPR(!UnrolledSource.empty());
+
         // Use serialization CI to be consistent with what will be saved in the archive.
         const auto SerializationCI = GetSerializationCI(ShaderCI);
         if (pRenderDeviceGL)
@@ -93,6 +107,11 @@ struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
         ShaderCI.ShaderCompiler = SHADER_COMPILER_DEFAULT;
         ShaderCI.Macros         = {}; // Macros are inlined into unrolled source
 
+        if (IsOptimized)
+        {
+            ShaderCI.SourceLanguage = SHADER_SOURCE_LANGUAGE_GLSL;
+            ShaderCI.EntryPoint     = "main";
+        }
         return ShaderCI;
     }
 
@@ -121,6 +140,88 @@ struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
         Source.append(UnrollShaderIncludes(CI));
         return Source;
     }
+
+    static String TransformSource(const ShaderCreateInfo&                      ShaderCI,
+                                  const ShaderGLImpl::CreateInfo&              GLShaderCI,
+                                  RENDER_DEVICE_TYPE                           DeviceType,
+                                  const SerializationDeviceImpl::GLProperties& GLProps)
+    {
+        std::string OptimizedGLSL;
+
+#if !DILIGENT_NO_GLSLANG
+        const std::string GLSLSourceString = ShaderGLImpl::BuildGLSLSourceString(
+            ShaderCI, GLShaderCI.DeviceInfo, GLShaderCI.AdapterInfo, GLProps.ZeroToOneClipZ);
+
+        GLSLangUtils::GLSLtoSPIRVAttribs Attribs;
+        Attribs.ShaderType = ShaderCI.Desc.ShaderType;
+        VERIFY_EXPR(DeviceType == RENDER_DEVICE_TYPE_GL || DeviceType == RENDER_DEVICE_TYPE_GLES);
+        Attribs.Version = DeviceType == RENDER_DEVICE_TYPE_GL ? GLSLangUtils::SpirvVersion::GL : GLSLangUtils::SpirvVersion::GLES;
+
+        Attribs.ppCompilerOutput = GLShaderCI.ppCompilerOutput;
+        Attribs.ShaderSource     = GLSLSourceString.c_str();
+        Attribs.SourceCodeLen    = static_cast<int>(GLSLSourceString.length());
+
+        const std::vector<unsigned int> SPIRV = GLSLangUtils::GLSLtoSPIRV(Attribs);
+        if (SPIRV.empty())
+            LOG_ERROR_AND_THROW("Failed to compile shader '", ShaderCI.Desc.Name, "'");
+
+        diligent_spirv_cross::CompilerGLSL Compiler{SPIRV};
+
+        diligent_spirv_cross::CompilerGLSL::Options Options;
+        if (DeviceType == RENDER_DEVICE_TYPE_GL)
+        {
+            if (ShaderCI.GLSLVersion != ShaderVersion{})
+                Options.version = ShaderCI.GLSLVersion.Major * 100 + ShaderCI.GLSLVersion.Minor * 10;
+            else
+                Options.version = 450;
+        }
+        else if (DeviceType == RENDER_DEVICE_TYPE_GLES)
+        {
+            if (ShaderCI.GLESSLVersion != ShaderVersion{})
+                Options.version = ShaderCI.GLESSLVersion.Major * 100 + ShaderCI.GLESSLVersion.Minor * 10;
+            else
+                Options.version = 310;
+            Options.es = true;
+        }
+        else
+        {
+            UNEXPECTED("Unexpected device type");
+        }
+        Options.separate_shader_objects = GLShaderCI.DeviceInfo.Features.SeparablePrograms;
+        // On some targets (WebGPU), uninitialized variables are banned.
+        Options.force_zero_initialized_variables = true;
+        // For opcodes where we have to perform explicit additional nan checks, very ugly code is generated.
+        Options.relax_nan_checks = true;
+
+        Options.fragment.default_float_precision = diligent_spirv_cross::CompilerGLSL::Options::Precision::DontCare;
+        Options.fragment.default_int_precision   = diligent_spirv_cross::CompilerGLSL::Options::Precision::DontCare;
+
+        Compiler.set_common_options(Options);
+
+        OptimizedGLSL = Compiler.compile();
+        if (OptimizedGLSL.empty())
+            LOG_ERROR_AND_THROW("Failed to generate GLSL for shader '", ShaderCI.Desc.Name, "'");
+
+        // Remove #version directive
+        const size_t VersionPos = OptimizedGLSL.find("#version");
+        if (VersionPos != std::string::npos)
+        {
+            const size_t NewLinePos = OptimizedGLSL.find('\n', VersionPos);
+            if (NewLinePos != std::string::npos)
+                OptimizedGLSL.erase(VersionPos, NewLinePos - VersionPos + 1);
+        }
+
+        SHADER_SOURCE_LANGUAGE SourceLang = ParseShaderSourceLanguageDefinition(GLSLSourceString);
+        if (SourceLang == SHADER_SOURCE_LANGUAGE_DEFAULT)
+        {
+            SourceLang = ShaderCI.SourceLanguage;
+        }
+
+        AppendShaderSourceLanguageDefinition(OptimizedGLSL, SourceLang);
+#endif
+
+        return OptimizedGLSL;
+    }
 };
 
 struct ShaderStageInfoGL
@@ -237,34 +338,8 @@ void SerializedShaderImpl::CreateShaderGL(IReferenceCounters*     pRefCounters,
         ppCompilerOutput == nullptr || *ppCompilerOutput == nullptr ? ppCompilerOutput : nullptr,
     };
 
-    CreateShader<CompiledShaderGL>(DeviceType::OpenGL, pRefCounters, ShaderCI, GLShaderCI, m_pDevice->GetRenderDevice(DeviceType));
-
-#if !DILIGENT_NO_GLSLANG
-    if (m_pDevice->GetGLProperties().ValidateShaders)
-    {
-        const auto* pCompiledShaderGL = GetShader<CompiledShaderGL>(DeviceObjectArchive::DeviceType::OpenGL);
-        VERIFY_EXPR(pCompiledShaderGL != nullptr);
-
-        const void* Source    = nullptr;
-        Uint64      SourceLen = 0;
-        // For OpenGL, GetBytecode returns the full GLSL source
-        pCompiledShaderGL->pShaderGL->GetBytecode(&Source, SourceLen);
-        VERIFY_EXPR(Source != nullptr && SourceLen != 0);
-
-        GLSLangUtils::GLSLtoSPIRVAttribs Attribs;
-
-        Attribs.ShaderType = ShaderCI.Desc.ShaderType;
-        VERIFY_EXPR(DeviceType == RENDER_DEVICE_TYPE_GL || DeviceType == RENDER_DEVICE_TYPE_GLES);
-        Attribs.Version = DeviceType == RENDER_DEVICE_TYPE_GL ? GLSLangUtils::SpirvVersion::GL : GLSLangUtils::SpirvVersion::GLES;
-
-        Attribs.ppCompilerOutput = ppCompilerOutput;
-        Attribs.ShaderSource     = static_cast<const char*>(Source);
-        Attribs.SourceCodeLen    = static_cast<int>(SourceLen);
-
-        if (GLSLangUtils::GLSLtoSPIRV(Attribs).empty())
-            LOG_ERROR_AND_THROW("Failed to compile shader '", ShaderCI.Desc.Name, "'");
-    }
-#endif
+    CreateShader<CompiledShaderGL>(DeviceType::OpenGL, pRefCounters, ShaderCI, GLShaderCI, m_pDevice->GetRenderDevice(DeviceType),
+                                   DeviceType, m_pDevice->GetGLProperties());
 }
 
 } // namespace Diligent

Graphics/Archiver/src/SerializationDeviceImpl.cpp

@@ -86,7 +86,8 @@ SerializationDeviceImpl::SerializationDeviceImpl(IReferenceCounters* pRefCounter
 
     if (m_ValidDeviceFlags & (ARCHIVE_DEVICE_DATA_FLAG_GL | ARCHIVE_DEVICE_DATA_FLAG_GLES))
     {
-        m_GLProps.ValidateShaders = CreateInfo.GL.ValidateShaders;
+        m_GLProps.OptimizeShaders = CreateInfo.GL.OptimizeShaders;
+        m_GLProps.ZeroToOneClipZ  = CreateInfo.GL.ZeroToOneClipZ;
     }
 
     if (m_ValidDeviceFlags & ARCHIVE_DEVICE_DATA_FLAG_VULKAN)

Graphics/GraphicsEngineOpenGL/include/ShaderGLImpl.hpp

@@ -85,6 +85,11 @@ class ShaderGLImpl final : public ShaderBase<EngineGLImplTraits>
         DataSize = m_GLSLSourceString.length();
     }
 
+    static std::string BuildGLSLSourceString(const ShaderCreateInfo&    ShaderCI,
+                                             const RenderDeviceInfo&    DeviceInfo,
+                                             const GraphicsAdapterInfo& AdapterInfo,
+                                             bool                       ZeroToOneClipZ);
+
 private:
     SHADER_SOURCE_LANGUAGE                   m_SourceLanguage = SHADER_SOURCE_LANGUAGE_DEFAULT;
     std::string                              m_GLSLSourceString;

Graphics/GraphicsEngineOpenGL/src/ShaderGLImpl.cpp

@@ -45,6 +45,32 @@ namespace Diligent
 
 constexpr INTERFACE_ID ShaderGLImpl::IID_InternalImpl;
 
+std::string ShaderGLImpl::BuildGLSLSourceString(const ShaderCreateInfo&    ShaderCI,
+                                                const RenderDeviceInfo&    DeviceInfo,
+                                                const GraphicsAdapterInfo& AdapterInfo,
+                                                bool                       ZeroToOneClipZ)
+{
+    if (ShaderCI.SourceLanguage == SHADER_SOURCE_LANGUAGE_GLSL_VERBATIM)
+    {
+        if (ShaderCI.Macros)
+        {
+            LOG_WARNING_MESSAGE("Shader macros are ignored when compiling GLSL verbatim in OpenGL backend");
+        }
+
+        // Read the source file directly and use it as is
+        ShaderSourceFileData SourceData = ReadShaderSourceFile(ShaderCI);
+        return std::string{SourceData.Source, SourceData.SourceLength};
+    }
+    else
+    {
+        static constexpr char NDCDefine[] = "#define _NDC_ZERO_TO_ONE 1\n";
+
+        // Build the full source code string that will contain GLSL version declaration,
+        // platform definitions, user-provided shader macros, etc.
+        return Diligent::BuildGLSLSourceString(ShaderCI, DeviceInfo, AdapterInfo, TargetGLSLCompiler::driver, (ZeroToOneClipZ ? NDCDefine : nullptr));
+    }
+}
+
 ShaderGLImpl::ShaderGLImpl(IReferenceCounters*     pRefCounters,
                            RenderDeviceGLImpl*     pDeviceGL,
                            const ShaderCreateInfo& ShaderCI,
@@ -70,32 +96,19 @@ ShaderGLImpl::ShaderGLImpl(IReferenceCounters*     pRefCounters,
     const auto& DeviceInfo  = GLShaderCI.DeviceInfo;
     const auto& AdapterInfo = GLShaderCI.AdapterInfo;
 
-    ShaderSourceFileData SourceData;
-    if (ShaderCI.SourceLanguage == SHADER_SOURCE_LANGUAGE_GLSL_VERBATIM)
-    {
-        if (ShaderCI.Macros)
-        {
-            LOG_WARNING_MESSAGE("Shader macros are ignored when compiling GLSL verbatim in OpenGL backend");
-        }
-
-        // Read the source file directly and use it as is
-        SourceData         = ReadShaderSourceFile(ShaderCI);
-        m_GLSLSourceString = std::string{SourceData.Source, SourceData.SourceLength};
+    m_GLSLSourceString = BuildGLSLSourceString(ShaderCI, DeviceInfo, AdapterInfo, DeviceInfo.NDC.MinZ == 0);
 
-        auto SourceLang = ParseShaderSourceLanguageDefinition(m_GLSLSourceString);
-        if (SourceLang != SHADER_SOURCE_LANGUAGE_DEFAULT)
-            m_SourceLanguage = SourceLang;
+    const SHADER_SOURCE_LANGUAGE SourceLang = ParseShaderSourceLanguageDefinition(m_GLSLSourceString);
+    if (SourceLang != SHADER_SOURCE_LANGUAGE_DEFAULT)
+    {
+        // Source language is already defined in the shader source (for instance,
+        // it may have been added by the archiver).
+        m_SourceLanguage = SourceLang;
     }
     else
     {
-        static constexpr char NDCDefine[] = "#define _NDC_ZERO_TO_ONE 1\n";
-
-        // Build the full source code string that will contain GLSL version declaration,
-        // platform definitions, user-provided shader macros, etc.
-        m_GLSLSourceString = BuildGLSLSourceString(
-            ShaderCI, DeviceInfo, AdapterInfo, TargetGLSLCompiler::driver,
-            (DeviceInfo.NDC.MinZ >= 0 ? NDCDefine : nullptr));
-
+        // Add source language definition to the shader source. It may be used by e.g.
+        // render state cache when packing shader source into the archive.
         AppendShaderSourceLanguageDefinition(m_GLSLSourceString, ShaderCI.SourceLanguage);
     }
 

Graphics/GraphicsTools/src/RenderStateCache.cpp

@@ -507,8 +507,7 @@ RenderStateCacheImpl::RenderStateCacheImpl(IReferenceCounters*               pRe
 
         case RENDER_DEVICE_TYPE_GL:
         case RENDER_DEVICE_TYPE_GLES:
-            // Do not validate shaders as compiling them is time-consuming
-            SerializationDeviceCI.GL.ValidateShaders = false;
+            SerializationDeviceCI.GL.ZeroToOneClipZ = SerializationDeviceCI.DeviceInfo.NDC.MinZ == 0;
             break;
 
         case RENDER_DEVICE_TYPE_VULKAN:

ThirdParty/CMakeLists.txt

@@ -84,7 +84,11 @@ if (VULKAN_SUPPORTED OR METAL_SUPPORTED)
         set(SPIRV_CROSS_ENABLE_TESTS OFF CACHE BOOL "Enable SPIRV-Cross tests.")
         set(SPIRV_CROSS_ENABLE_MSL ${METAL_SUPPORTED} CACHE BOOL "Enable MSL target support.")
         # MSL support requires GLSL
-        set(SPIRV_CROSS_ENABLE_GLSL ${SPIRV_CROSS_ENABLE_MSL} CACHE BOOL "Enable GLSL support.")
+        if (${SPIRV_CROSS_ENABLE_MSL} OR ${GL_SUPPORTED} OR ${GLES_SUPPORTED})
+            set(SPIRV_CROSS_ENABLE_GLSL TRUE CACHE BOOL "Enable GLSL support.")
+        else()
+            set(SPIRV_CROSS_ENABLE_GLSL OFF CACHE BOOL "Enable GLSL support.")
+        endif()
         set(SPIRV_CROSS_ENABLE_HLSL OFF CACHE BOOL "Enable HLSL target support.")
         set(SPIRV_CROSS_ENABLE_CPP OFF CACHE BOOL "Enable C++ target support.")
         set(SPIRV_CROSS_ENABLE_REFLECT OFF CACHE BOOL "Enable JSON reflection target support.")