Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PmtYaml: Parser fixes #477

Merged
merged 1 commit into from
Dec 5, 2024
Merged

PmtYaml: Parser fixes #477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 47 additions & 50 deletions core/include/gnuradio-4.0/YamlPmt.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ struct ParseContext {

bool atEndOfDocument() const { return lineIdx == lines.size(); }

std::size_t currentIndent() const { return lines[lineIdx].find_first_not_of(' '); }
std::size_t currentIndent(std::string_view indentChars = " ") const { return lines[lineIdx].find_first_not_of(indentChars); }

ParseError makeError(std::string message) const { return {.line = lineIdx + 1, .column = columnIdx + 1, .message = std::move(message)}; }

Expand Down Expand Up @@ -667,9 +667,16 @@ inline std::expected<pmtv::pmt, ParseError> parsePlainScalar(ParseContext& ctx,
}

inline std::expected<pmtv::pmt, ParseError> parseScalar(ParseContext& ctx, std::string_view typeTag, int currentIndentLevel) {
// remove leading spaces
ctx.consumeSpaces();
const auto initialLine = ctx.lineIdx;
ctx.consumeWhitespaceAndComments();

if (ctx.atEndOfDocument()) {
return std::monostate{};
}
const auto skippedLines = ctx.lineIdx > initialLine;
if (skippedLines && currentIndentLevel >= 0 && ctx.currentIndent() <= static_cast<std::size_t>(currentIndentLevel)) {
return std::monostate{};
}
// handle multi-line indicators '|', '|-', '>', '>-'
if ((typeTag == "!!str" || typeTag.empty()) && (!ctx.atEndOfLine() && (ctx.front() == '|' || ctx.front() == '>'))) {
char indicator = ctx.front();
Expand Down Expand Up @@ -760,32 +767,6 @@ inline std::expected<pmtv::pmt, ParseError> parseScalar(ParseContext& ctx, std::

enum class ValueType { List, Map, Scalar };

inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) {
ctx.consumeSpaces();
if (ctx.startsWith("[")) {
return ValueType::List;
}
if (ctx.startsWith("{")) {
return ValueType::Map;
}
if (!ctx.atEndOfLine()) {
return ValueType::Scalar;
}
ctx.skipToNextLine();
while (!ctx.atEndOfDocument()) {
ctx.consumeWhitespaceAndComments();
if (ctx.atEndOfDocument() || (previousIndent >= 0 && ctx.currentIndent() <= static_cast<std::size_t>(previousIndent))) {
break;
}
ctx.consumeSpaces();
if (ctx.startsWith("-")) {
return ValueType::List;
}
return ValueType::Map;
}
return ValueType::Scalar;
}

inline std::expected<std::string, ParseError> parseKey(ParseContext& ctx, std::string_view extraDelimiters = {}) {
ctx.consumeSpaces();

Expand Down Expand Up @@ -823,6 +804,30 @@ inline std::expected<std::string, ParseError> parseKey(ParseContext& ctx, std::s
return key;
}

inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) {
const auto initialLine = ctx.lineIdx;
ctx.consumeWhitespaceAndComments();

if (ctx.atEndOfDocument()) {
return ValueType::Scalar;
}

const auto skippedLines = ctx.lineIdx > initialLine;
if (skippedLines && previousIndent >= 0 && ctx.currentIndent() <= static_cast<std::size_t>(previousIndent)) {
return ValueType::Scalar;
}
if (ctx.startsWith("[") || (ctx.startsWith("- ") || ctx.remainingLine() == "-")) {
return ValueType::List;
}

if (ctx.startsWith("{")) {
return ValueType::Map;
}

const auto key = parseKey(ctx);
return key.has_value() ? ValueType::Map : ValueType::Scalar;
}

template<typename T>
struct ConvertList {
pmtv::pmt operator()(const std::vector<pmtv::pmt>& list) {
Expand Down Expand Up @@ -931,32 +936,31 @@ inline auto parseFlow(ParseContext& ctx, std::string_view typeTag, int parentInd
}

inline std::expected<pmtv::map_t, ParseError> parseMap(ParseContext& ctx, int parentIndentLevel) {
ctx.consumeWhitespaceAndComments();
if (ctx.consumeIfStartsWith("{")) {
auto map = parseFlow<FlowType::Map>(ctx, "", parentIndentLevel);
ctx.skipToNextLine();
return map;
}

pmtv::map_t map;
bool firstLine = true;

while (!ctx.atEndOfDocument()) {
if (ctx.startsWith("---")) {
return std::unexpected(ctx.makeError("Parser limitation: Multiple documents not supported"));
}
ctx.consumeSpaces();
if (ctx.atEndOfLine() || ctx.startsWith("#")) {
// skip empty lines and comments
ctx.skipToNextLine();
continue;
}
ctx.consumeWhitespaceAndComments();

const auto line_indent = ctx.currentIndent();
const auto line_indent = firstLine ? ctx.currentIndent(" -") : ctx.currentIndent(); // Ignore "-" if map is in a list

if (parentIndentLevel >= 0 && line_indent <= static_cast<std::size_t>(parentIndentLevel)) {
// indentation decreased; end of current map
break;
}

firstLine = false;

const auto maybeKey = parseKey(ctx);
if (!maybeKey.has_value()) {
return std::unexpected(maybeKey.error());
Expand Down Expand Up @@ -1009,6 +1013,7 @@ inline std::expected<pmtv::map_t, ParseError> parseMap(ParseContext& ctx, int pa
}

inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::string_view typeTag, int parentIndentLevel) {
ctx.consumeWhitespaceAndComments();
if (ctx.consumeIfStartsWith("[")) {
auto l = parseFlow<FlowType::List>(ctx, typeTag, parentIndentLevel);
ctx.skipToNextLine();
Expand All @@ -1017,32 +1022,24 @@ inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::st

std::vector<pmtv::pmt> list;

ctx.skipToNextLine();

while (!ctx.atEndOfDocument()) {
ctx.consumeSpaces();
if (ctx.atEndOfLine() || ctx.startsWith("#")) {
// skip empty lines and comments
ctx.skipToNextLine();
continue;
}
ctx.consumeWhitespaceAndComments();

const std::size_t line_indent = ctx.currentIndent();
if (parentIndentLevel >= 0 && line_indent <= static_cast<size_t>(parentIndentLevel)) {
// indentation decreased; end of current list
break;
}

ctx.consumeSpaces();

if (!ctx.consumeIfStartsWith('-')) {
if (!ctx.consumeIfStartsWith("-")) {
// not a list item
return std::unexpected(ctx.makeError("Expected list item"));
}

ctx.consumeSpaces();

const auto tagPos = ctx.columnIdx;
const auto itemIndent = ctx.columnIdx;

const auto maybeLocalTag = parseTag(ctx);
if (!maybeLocalTag.has_value()) {
return std::unexpected(maybeLocalTag.error());
Expand All @@ -1060,7 +1057,7 @@ inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::st
switch (peekedType) {
case ValueType::List: {
if (!typeTag.empty()) {
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for list containing lists", tagPos));
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for list containing lists", itemIndent));
}
auto parsedValue = parseList(ctx, tag, static_cast<int>(line_indent));
if (!parsedValue.has_value()) {
Expand All @@ -1071,7 +1068,7 @@ inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::st
}
case ValueType::Map: {
if (!localTag.empty()) {
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for maps", tagPos));
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for maps", itemIndent));
}
auto parsedValue = parseMap(ctx, static_cast<int>(line_indent));
if (!parsedValue.has_value()) {
Expand Down
65 changes: 64 additions & 1 deletion core/test/qa_YamlPmt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ void testYAML(std::string_view src, const pmtv::map_t expected) {
} else {
fmt::println(std::cerr, "Unexpected: {}", formatResult(deserializedMap));
expect(false);
return;
}

// Then test that serializing and deserializing the map again results in the same map
Expand All @@ -141,20 +142,32 @@ const boost::ut::suite YamlPmtTests = [] {
using namespace std::string_literals;
using namespace std::string_view_literals;

"Comments"_test = [] {
"Comments and Whitespace"_test = [] {
constexpr std::string_view src1 = R"(# Comment
double: !!float64 42 # Comment
string: "#Hello" # Comment
null: # Comment
#Comment: 43
# string: | # Comment
# Hello
number:
42
list:
[]
list2: [ 42
]
map:
{}
)";

pmtv::map_t expected;
expected["double"] = 42.0;
expected["string"] = "#Hello";
expected["null"] = std::monostate{};
expected["number"] = static_cast<int64_t>(42);
expected["list"] = std::vector<pmtv::pmt>{};
expected["list2"] = std::vector<pmtv::pmt>{static_cast<int64_t>(42)};
expected["map"] = pmtv::map_t{};

testYAML(src1, expected);
};
Expand Down Expand Up @@ -458,6 +471,8 @@ nullVector: !!null
- null
emptyVector: !!str []
emptyPmtVector: []
emptyAfterNewline:
[]
flowDouble: !!float64 [1, 2, 3]
flowString: !!str ["Hello, ", "World", "Multiple\nlines"]
flowMultiline: !!str [ "Hello, " , #]
Expand All @@ -478,6 +493,14 @@ flowMultiline: !!str [ "Hello, " , #]
- !!str [3, 4]
- { key: !!str [5, 6] }
nestedFlow: [ !!str [1, 2], [3, 4] ]
vectorWithBlockMap:
- name: ArraySink<double>
id: ArraySink
parameters:
name: Block
vectorWithColons:
- "key: value"
- "key2: value2"
)";

pmtv::map_t expected;
Expand All @@ -492,12 +515,15 @@ nestedFlow: [ !!str [1, 2], [3, 4] ]
expected["nullVector"] = std::monostate{};
expected["emptyVector"] = std::vector<std::string>{};
expected["emptyPmtVector"] = std::vector<pmtv::pmt>{};
expected["emptyAfterNewline"] = std::vector<pmtv::pmt>{};
expected["flowDouble"] = std::vector<double>{1.0, 2.0, 3.0};
expected["flowString"] = std::vector<std::string>{"Hello, ", "World", "Multiple\nlines"};
expected["flowMultiline"] = std::vector<std::string>{"Hello, ", "][", "World", "Multiple\nlines"};
expected["nestedVector"] = std::vector<pmtv::pmt>{std::vector<std::string>{"1", "2"}, std::vector<pmtv::pmt>{static_cast<int64_t>(3), static_cast<int64_t>(4)}};
expected["nestedFlow"] = std::vector<pmtv::pmt>{std::vector<std::string>{"1", "2"}, std::vector<pmtv::pmt>{static_cast<int64_t>(3), static_cast<int64_t>(4)}};
expected["nestedVector2"] = std::vector<pmtv::pmt>{static_cast<int64_t>(42), std::vector<std::string>{"1", "2"}, std::vector<std::string>{"3", "4"}, pmtv::map_t{{"key", std::vector<std::string>{"5", "6"}}}};
expected["vectorWithBlockMap"] = std::vector<pmtv::pmt>{pmtv::map_t{{"name", "ArraySink<double>"}, {"id", "ArraySink"}, {"parameters", pmtv::map_t{{"name", "Block"}}}}};
expected["vectorWithColons"] = std::vector<pmtv::pmt>{"key: value", "key2: value2"};

testYAML(src1, expected);

Expand Down Expand Up @@ -623,6 +649,43 @@ key#Comment: foo
expect(eq(formatResult(yaml::deserialize(invalidKeyComment)), "Error in 3:1: Could not find key/value separator ':'"sv));
};

"GRC"_test = [] {
constexpr std::string_view src = R"(
blocks:
- name: ArraySink<double>
id: ArraySink
parameters:
name: ArraySink<double>
- name: ArraySource<double>
id: ArraySource
parameters:
name: ArraySource<double>
connections:
- [ArraySource<double>, [0, 0], ArraySink<double>, [1, 1]]
- [ArraySource<double>, [0, 1], ArraySink<double>, [1, 0]]
- [ArraySource<double>, [1, 0], ArraySink<double>, [0, 0]]
- [ArraySource<double>, [1, 1], ArraySink<double>, [0, 1]]
)";

pmtv::map_t expected;
pmtv::map_t block1;
block1["name"] = "ArraySink<double>";
block1["id"] = "ArraySink";
block1["parameters"] = pmtv::map_t{{"name", "ArraySink<double>"}};
pmtv::map_t block2;
block2["name"] = "ArraySource<double>";
block2["id"] = "ArraySource";
block2["parameters"] = pmtv::map_t{{"name", "ArraySource<double>"}};
expected["blocks"] = std::vector<pmtv::pmt>{block1, block2};
const auto zero = pmtv::pmt{static_cast<int64_t>(0)};
const auto one = pmtv::pmt{static_cast<int64_t>(1)};
using PmtVec = std::vector<pmtv::pmt>;

expected["connections"] = std::vector<pmtv::pmt>{PmtVec{"ArraySource<double>", std::vector{zero, zero}, "ArraySink<double>", std::vector{one, one}}, PmtVec{"ArraySource<double>", std::vector{zero, one}, "ArraySink<double>", std::vector{one, zero}}, PmtVec{"ArraySource<double>", std::vector{one, zero}, "ArraySink<double>", std::vector{zero, zero}}, PmtVec{"ArraySource<double>", std::vector{one, one}, "ArraySink<double>", std::vector{zero, one}}};

testYAML(src, expected);
};

"Complex"_test = [] {
constexpr std::string_view src = R"(
complex: !!complex64 (1.0, -1.0)
Expand Down
Loading