Skip to content

Commit

Permalink
HLSL2GLSL Converter: handle preprocessor-defined shader function retu…
Browse files Browse the repository at this point in the history 
…rn type
  • Loading branch information
@TheMostDiligent
TheMostDiligent committed Jan 10, 2024
1 parent 431f4ee commit 06cfb24
Show file tree
Hide file tree
Showing 4 changed files with 213 additions and 55 deletions.
19 changes: 14 additions & 5 deletions Graphics/HLSL2GLSLConverterLib/include/HLSL2GLSLConverterImpl.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2019-2022 Diligent Graphics LLC
* Copyright 2019-2024 Diligent Graphics LLC
* Copyright 2015-2019 Egor Yusov
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -257,6 +257,7 @@ class HLSL2GLSLConverterImpl
TokenType Type = TokenType::Undefined;
String Literal;
String Delimiter;
size_t Idx = ~size_t{0};

void SetType(TokenType _Type)
{
Expand Down Expand Up @@ -303,19 +304,22 @@ class HLSL2GLSLConverterImpl
const std::string::const_iterator& DelimStart,
const std::string::const_iterator& DelimEnd,
const std::string::const_iterator& LiteralStart,
const std::string::const_iterator& LiteralEnd)
const std::string::const_iterator& LiteralEnd,
size_t Idx)
{
return TokenInfo{_Type, std::string{LiteralStart, LiteralEnd}, std::string{DelimStart, DelimEnd}};
return TokenInfo{_Type, std::string{LiteralStart, LiteralEnd}, std::string{DelimStart, DelimEnd}, Idx};
}

TokenInfo() {}

TokenInfo(TokenType _Type,
std::string _Literal,
std::string _Delimiter = "") :
std::string _Delimiter = "",
size_t _Idx = ~size_t{0}) :
Type{_Type},
Literal{std::move(_Literal)},
Delimiter{std::move(_Delimiter)}
Delimiter{std::move(_Delimiter)},
Idx{_Idx}
{}

size_t GetDelimiterLen() const
Expand Down Expand Up @@ -402,6 +406,8 @@ class HLSL2GLSLConverterImpl

const HLSLObjectInfo* FindHLSLObject(const String& Name);

void ParseGlobalPreprocessorDefines();

void ProcessShaderDeclaration(TokenListType::iterator EntryPointToken, SHADER_TYPE ShaderType);

void ProcessObjectMethods(const TokenListType::iterator& ScopeStart, const TokenListType::iterator& ScopeEnd);
Expand Down Expand Up @@ -586,6 +592,9 @@ class HLSL2GLSLConverterImpl
// List of tokens defining structs
std::unordered_map<HashMapStringKey, TokenListType::iterator> m_StructDefinitions;

// List of preprocessor macro definitions in global scope
std::unordered_map<HashMapStringKey, TokenListType::iterator> m_PreprocessorDefinitions;

// Stack of parsed objects, for every scope level.
// There are currently only two levels:
// level 0 - global scope, contains all global objects
Expand Down
214 changes: 165 additions & 49 deletions Graphics/HLSL2GLSLConverterLib/src/HLSL2GLSLConverterImpl.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2019-2022 Diligent Graphics LLC
* Copyright 2019-2024 Diligent Graphics LLC
* Copyright 2015-2019 Egor Yusov
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -813,8 +813,18 @@ void HLSL2GLSLConverterImpl::ConversionStream::InsertIncludes(String& GLSLSource
// The function converts source code into a token list
void HLSL2GLSLConverterImpl::ConversionStream::Tokenize(const String& Source)
{
size_t TokenIdx = 0;

m_Tokens = Parsing::Tokenize<TokenInfo, decltype(m_Tokens)>(
Source.begin(), Source.end(), TokenInfo::Create,
Source.begin(), Source.end(),
[&TokenIdx](TokenType Type,
const std::string::const_iterator& DelimStart,
const std::string::const_iterator& DelimEnd,
const std::string::const_iterator& LiteralStart,
const std::string::const_iterator& LiteralEnd) //
{
return TokenInfo::Create(Type, DelimStart, DelimEnd, LiteralStart, LiteralEnd, TokenIdx++);
},
[&](const std::string::const_iterator& Start, const std::string::const_iterator& End) //
{
auto KeywordIt = m_Converter.m_HLSLKeywords.find(HashMapStringKey{std::string{Start, End}});
Expand All @@ -827,6 +837,71 @@ void HLSL2GLSLConverterImpl::ConversionStream::Tokenize(const String& Source)
});
}

void HLSL2GLSLConverterImpl::ConversionStream::ParseGlobalPreprocessorDefines()
{
auto Token = m_Tokens.begin();

// Collect global-scope preprocessor definitions
int PreprocessorScopeLevel = 0;
while (Token != m_Tokens.end())
{
if (Token->Type != TokenType::PreprocessorDirective)
{
++Token;
continue;
}

const auto Directive = RefinePreprocessorDirective(Token->Literal);

if (Directive == "if" ||
Directive == "ifdef" ||
Directive == "ifndef")
{
++PreprocessorScopeLevel;
}
else if (Directive == "endif")
{
if (PreprocessorScopeLevel > 0)
{
--PreprocessorScopeLevel;
}
else
{
LOG_ERROR_MESSAGE("No matching #if directive\n", PrintTokenContext(Token, 4));
}
}
else if (Directive == "define")
{
// #define MACRO
// ^

// Only process macros in the global scope as we don't
// handle conditional compilation
if (PreprocessorScopeLevel == 0)
{
auto MacroNameToken = Token;
++MacroNameToken;
// #define MACRO
// ^
if (MacroNameToken != m_Tokens.end() &&
// The name should not be empty
!MacroNameToken->Literal.empty() &&
// Check that the name is on the same line
MacroNameToken->Delimiter.find_first_of("\r\n") == std::string::npos)
{
m_PreprocessorDefinitions.emplace(MacroNameToken->Literal, Token);
}
}
}

++Token;
}

if (PreprocessorScopeLevel > 0)
{
LOG_ERROR_MESSAGE("Missing #endif directive at the end of the file. Current preprocessor scope level: ", PreprocessorScopeLevel);
}
}

// The function replaces cbuffer with uniform and adds semicolon if it is missing after the closing brace:
// cbuffer
Expand Down Expand Up @@ -2270,60 +2345,63 @@ void HLSL2GLSLConverterImpl::ConversionStream::ParseShaderParameter(TokenListTyp
auto TypeToken = Token;
ParamInfo.Type = Token->Literal;

++Token;
// out float4 Color : SV_Target,
// ^
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF while parsing argument list");
VERIFY_PARSER_STATE(Token, Token->Type == TokenType::Identifier, "Missing argument name after ", ParamInfo.Type);
ParamInfo.Name = Token->Literal;

++Token;
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF");

if (Token->Type == TokenType::OpenSquareBracket)
if (ParamInfo.storageQualifier != ShaderParameterInfo::StorageQualifier::Ret)
{
// triangle VSOut In[3]
// ^
ProcessScope(
Token, m_Tokens.end(), TokenType::OpenSquareBracket, TokenType::ClosingSquareBracket,
[&](TokenListType::iterator& tkn, int) {
ParamInfo.ArraySize.append(tkn->Delimiter);
ParamInfo.ArraySize.append(tkn->Literal);
++tkn;
} //
);
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF");
// triangle VSOut In[3],
// ^
VERIFY_PARSER_STATE(Token, Token->Type == TokenType::ClosingSquareBracket, "Closing staple expected");
++Token;
// out float4 Color : SV_Target,
// ^
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF while parsing argument list");
VERIFY_PARSER_STATE(Token, Token->Type == TokenType::Identifier, "Missing argument name after ", ParamInfo.Type);
ParamInfo.Name = Token->Literal;

++Token;
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF");
VERIFY_PARSER_STATE(Token, Token->Type != TokenType::OpenSquareBracket, "Multi-dimensional arrays are not supported");
}


if (TypeToken->IsBuiltInType())
{
// out float4 Color : SV_Target,
// ^
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected end of file after argument \"", ParamInfo.Name, '\"');
if (Token->Literal == ":")
if (Token->Type == TokenType::OpenSquareBracket)
{
++Token;
// out float4 Color : SV_Target,
// ^
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected end of file while looking for semantic for argument \"", ParamInfo.Name, '\"');
VERIFY_PARSER_STATE(Token, Token->Type == TokenType::Identifier, "Missing semantic for argument \"", ParamInfo.Name, '\"');
// Transform to lower case - semantics are case-insensitive
ParamInfo.Semantic = StrToLower(Token->Literal);
// triangle VSOut In[3]
// ^
ProcessScope(
Token, m_Tokens.end(), TokenType::OpenSquareBracket, TokenType::ClosingSquareBracket,
[&](TokenListType::iterator& tkn, int) {
ParamInfo.ArraySize.append(tkn->Delimiter);
ParamInfo.ArraySize.append(tkn->Literal);
++tkn;
} //
);
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF");
// triangle VSOut In[3],
// ^
VERIFY_PARSER_STATE(Token, Token->Type == TokenType::ClosingSquareBracket, "Closing staple expected");

++Token;
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected EOF");
VERIFY_PARSER_STATE(Token, Token->Type != TokenType::OpenSquareBracket, "Multi-dimensional arrays are not supported");
}

if (TypeToken->IsBuiltInType())
{
// out float4 Color : SV_Target,
// ^
// ^
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected end of file after argument \"", ParamInfo.Name, '\"');
if (Token->Literal == ":")
{
++Token;
// out float4 Color : SV_Target,
// ^
VERIFY_PARSER_STATE(Token, Token != m_Tokens.end(), "Unexpected end of file while looking for semantic for argument \"", ParamInfo.Name, '\"');
VERIFY_PARSER_STATE(Token, Token->Type == TokenType::Identifier, "Missing semantic for argument \"", ParamInfo.Name, '\"');
// Transform to lower case - semantics are case-insensitive
ParamInfo.Semantic = StrToLower(Token->Literal);

++Token;
// out float4 Color : SV_Target,
// ^
}
}
}
else

if (!TypeToken->IsBuiltInType())
{
const auto& StructName = TypeToken->Literal;
auto it = m_StructDefinitions.find(StructName.c_str());
Expand All @@ -2349,8 +2427,11 @@ void HLSL2GLSLConverterImpl::ConversionStream::ParseShaderParameter(TokenListTyp
while (TypeToken != m_Tokens.end() && TypeToken->Type != TokenType::ClosingBrace)
{
ShaderParameterInfo MemberInfo;
MemberInfo.storageQualifier = ParamInfo.storageQualifier;
ParseShaderParameter(TypeToken, MemberInfo);
// Set storage qualifier after we process the member as otherwise
// members of a struct with the StorageQualifier::Ret qualifier
// will not be processed.
MemberInfo.storageQualifier = ParamInfo.storageQualifier;
ParamInfo.members.emplace_back(std::move(MemberInfo));
// struct VSOutput
// {
Expand All @@ -2376,11 +2457,43 @@ void HLSL2GLSLConverterImpl::ConversionStream::ProcessFunctionParameters(TokenLi
// ^
auto FuncNameToken = Token;

bIsVoid = TypeToken->Type == TokenType::kw_void;
auto ActualTypeToken = TypeToken;
{
// PS_OUTPUT TestPS ( in VSOutput In,

auto define_it = m_PreprocessorDefinitions.find(ActualTypeToken->Literal.c_str());
if (define_it != m_PreprocessorDefinitions.end())
{
auto DefinedTypeToken = define_it->second;
// #define PS_OUTPUT PSOutput
// ^
// DefinedTypeToken

// Check that the define directive is before the type token
if (DefinedTypeToken->Idx < TypeToken->Idx)
{
++DefinedTypeToken;
// #define PS_OUTPUT PSOutput
// ^
// DefinedTypeToken
if (DefinedTypeToken != m_Tokens.end() && DefinedTypeToken->Literal == TypeToken->Literal)
{
++DefinedTypeToken;
// #define PS_OUTPUT PSOutput
// ^
// DefinedTypeToken
if (DefinedTypeToken != m_Tokens.end() && DefinedTypeToken->Type == TokenType::Identifier)
ActualTypeToken = DefinedTypeToken;
}
}
}
}

bIsVoid = ActualTypeToken->Type == TokenType::kw_void;
if (!bIsVoid)
{
ShaderParameterInfo RetParam;
RetParam.Type = TypeToken->Literal;
RetParam.Type = ActualTypeToken->Literal;
RetParam.Name = FuncNameToken->Literal;
RetParam.storageQualifier = ShaderParameterInfo::StorageQualifier::Ret;
Params.emplace_back(std::move(RetParam));
Expand Down Expand Up @@ -2637,7 +2750,7 @@ void HLSL2GLSLConverterImpl::ConversionStream::ProcessFunctionParameters(TokenLi
else
{
// VSOut TestVS ()
auto TmpTypeToken = TypeToken;
auto TmpTypeToken = ActualTypeToken;
ParseShaderParameter(TmpTypeToken, RetParam);
}
TypeToken->Type = TokenType::Identifier;
Expand Down Expand Up @@ -4482,6 +4595,8 @@ String HLSL2GLSLConverterImpl::ConversionStream::Convert(const Char* EntryPoint,
}
}

ParseGlobalPreprocessorDefines();

auto ShaderEntryPointToken = m_Tokens.end();
// Process textures and search for the shader entry point.
// GLSL does not allow local variables of sampler type, so the
Expand Down Expand Up @@ -4665,6 +4780,7 @@ String HLSL2GLSLConverterImpl::ConversionStream::Convert(const Char* EntryPoint,
{
m_Tokens.swap(TokensCopy);
m_StructDefinitions.clear();
m_PreprocessorDefinitions.clear();
m_Objects.clear();
}

Expand Down
25 changes: 25 additions & 0 deletions Tests/DiligentCoreAPITest/assets/shaders/HLSL2GLSLConverter/PreprocessorTest.hlsl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
struct PSOutput
{
float4 Color : SV_Target0;
};

struct PSInput
{
float4 Pos : SV_Position;
};

# define PS_OUTPUT PSOutput
# define PS_INPUT PSInput

#ifdef MACRO0
# define PS_OUTPUT abc
#elif defined(MACRO1)
# define PS_OUTPUT xyz
#endif

PS_OUTPUT main(in PSInput PSIn)
{
PS_OUTPUT PSOut;
PSOut.Color = float4(PSIn.Pos.xy, 0.0, 0.0);
return PSOut;
}
10 changes: 9 additions & 1 deletion Tests/DiligentCoreAPITest/src/HLSL2GLSLConverterTest.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2019-2023 Diligent Graphics LLC
* Copyright 2019-2024 Diligent Graphics LLC
* Copyright 2015-2019 Egor Yusov
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -142,4 +142,12 @@ TEST(HLSL2GLSLConverterTest, GS)
EXPECT_NE(pGS, nullptr);
}

TEST(HLSL2GLSLConverterTest, Preprocessor)
{
GPUTestingEnvironment::ScopedReset EnvironmentAutoReset;

auto pPS = CreateTestShader("PreprocessorTest.hlsl", "main", SHADER_TYPE_PIXEL);
EXPECT_NE(pPS, nullptr);
}

} // namespace

0 comments on commit 06cfb24

Please sign in to comment.