Skip to content

Commit

Permalink
PmtYaml: Parser fixes
Browse files Browse the repository at this point in the history 
 * Make sure block-style maps in block-style lists are parsed
   correctly. This is needed to parse our existing GRC files.
 * Support unexpected whitespace/newlines between key and scalars
   as well as flow-style maps/lists.

Signed-off-by: Frank Osterfeld <[email protected]>
  • Loading branch information
@frankosterfeld
frankosterfeld committed Dec 4, 2024
1 parent 4829478 commit e4597a6
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 51 deletions.
97 changes: 47 additions & 50 deletions core/include/gnuradio-4.0/YamlPmt.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ struct ParseContext {

bool atEndOfDocument() const { return lineIdx == lines.size(); }

std::size_t currentIndent() const { return lines[lineIdx].find_first_not_of(' '); }
std::size_t currentIndent(std::string_view indentChars = " ") const { return lines[lineIdx].find_first_not_of(indentChars); }

ParseError makeError(std::string message) const { return {.line = lineIdx + 1, .column = columnIdx + 1, .message = std::move(message)}; }

Expand Down Expand Up @@ -667,9 +667,16 @@ inline std::expected<pmtv::pmt, ParseError> parsePlainScalar(ParseContext& ctx,
}

inline std::expected<pmtv::pmt, ParseError> parseScalar(ParseContext& ctx, std::string_view typeTag, int currentIndentLevel) {
// remove leading spaces
ctx.consumeSpaces();
const auto initialLine = ctx.lineIdx;
ctx.consumeWhitespaceAndComments();

if (ctx.atEndOfDocument()) {
return std::monostate{};
}
const auto skippedLines = ctx.lineIdx > initialLine;
if (skippedLines && currentIndentLevel >= 0 && ctx.currentIndent() <= static_cast<std::size_t>(currentIndentLevel)) {
return std::monostate{};
}
// handle multi-line indicators '|', '|-', '>', '>-'
if ((typeTag == "!!str" || typeTag.empty()) && (!ctx.atEndOfLine() && (ctx.front() == '|' || ctx.front() == '>'))) {
char indicator = ctx.front();
Expand Down Expand Up @@ -760,32 +767,6 @@ inline std::expected<pmtv::pmt, ParseError> parseScalar(ParseContext& ctx, std::

enum class ValueType { List, Map, Scalar };

inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) {
ctx.consumeSpaces();
if (ctx.startsWith("[")) {
return ValueType::List;
}
if (ctx.startsWith("{")) {
return ValueType::Map;
}
if (!ctx.atEndOfLine()) {
return ValueType::Scalar;
}
ctx.skipToNextLine();
while (!ctx.atEndOfDocument()) {
ctx.consumeWhitespaceAndComments();
if (ctx.atEndOfDocument() || (previousIndent >= 0 && ctx.currentIndent() <= static_cast<std::size_t>(previousIndent))) {
break;
}
ctx.consumeSpaces();
if (ctx.startsWith("-")) {
return ValueType::List;
}
return ValueType::Map;
}
return ValueType::Scalar;
}

inline std::expected<std::string, ParseError> parseKey(ParseContext& ctx, std::string_view extraDelimiters = {}) {
ctx.consumeSpaces();

Expand Down Expand Up @@ -823,6 +804,30 @@ inline std::expected<std::string, ParseError> parseKey(ParseContext& ctx, std::s
return key;
}

inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) {
const auto initialLine = ctx.lineIdx;
ctx.consumeWhitespaceAndComments();

if (ctx.atEndOfDocument()) {
return ValueType::Scalar;
}

const auto skippedLines = ctx.lineIdx > initialLine;
if (skippedLines && previousIndent >= 0 && ctx.currentIndent() <= static_cast<std::size_t>(previousIndent)) {
return ValueType::Scalar;
}
if (ctx.startsWith("[") || (ctx.startsWith("- ") || ctx.remainingLine() == "-")) {
return ValueType::List;
}

if (ctx.startsWith("{")) {
return ValueType::Map;
}

const auto key = parseKey(ctx);
return key.has_value() ? ValueType::Map : ValueType::Scalar;
}

template<typename T>
struct ConvertList {
pmtv::pmt operator()(const std::vector<pmtv::pmt>& list) {
Expand Down Expand Up @@ -931,32 +936,31 @@ inline auto parseFlow(ParseContext& ctx, std::string_view typeTag, int parentInd
}

inline std::expected<pmtv::map_t, ParseError> parseMap(ParseContext& ctx, int parentIndentLevel) {
ctx.consumeWhitespaceAndComments();
if (ctx.consumeIfStartsWith("{")) {
auto map = parseFlow<FlowType::Map>(ctx, "", parentIndentLevel);
ctx.skipToNextLine();
return map;
}

pmtv::map_t map;
bool firstLine = true;

while (!ctx.atEndOfDocument()) {
if (ctx.startsWith("---")) {
return std::unexpected(ctx.makeError("Parser limitation: Multiple documents not supported"));
}
ctx.consumeSpaces();
if (ctx.atEndOfLine() || ctx.startsWith("#")) {
// skip empty lines and comments
ctx.skipToNextLine();
continue;
}
ctx.consumeWhitespaceAndComments();

const auto line_indent = ctx.currentIndent();
const auto line_indent = firstLine ? ctx.currentIndent(" -") : ctx.currentIndent(); // Ignore "-" if map is in a list

if (parentIndentLevel >= 0 && line_indent <= static_cast<std::size_t>(parentIndentLevel)) {
// indentation decreased; end of current map
break;
}

firstLine = false;

const auto maybeKey = parseKey(ctx);
if (!maybeKey.has_value()) {
return std::unexpected(maybeKey.error());
Expand Down Expand Up @@ -1009,6 +1013,7 @@ inline std::expected<pmtv::map_t, ParseError> parseMap(ParseContext& ctx, int pa
}

inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::string_view typeTag, int parentIndentLevel) {
ctx.consumeWhitespaceAndComments();
if (ctx.consumeIfStartsWith("[")) {
auto l = parseFlow<FlowType::List>(ctx, typeTag, parentIndentLevel);
ctx.skipToNextLine();
Expand All @@ -1017,32 +1022,24 @@ inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::st

std::vector<pmtv::pmt> list;

ctx.skipToNextLine();

while (!ctx.atEndOfDocument()) {
ctx.consumeSpaces();
if (ctx.atEndOfLine() || ctx.startsWith("#")) {
// skip empty lines and comments
ctx.skipToNextLine();
continue;
}
ctx.consumeWhitespaceAndComments();

const std::size_t line_indent = ctx.currentIndent();
if (parentIndentLevel >= 0 && line_indent <= static_cast<size_t>(parentIndentLevel)) {
// indentation decreased; end of current list
break;
}

ctx.consumeSpaces();

if (!ctx.consumeIfStartsWith('-')) {
if (!ctx.consumeIfStartsWith("-")) {
// not a list item
return std::unexpected(ctx.makeError("Expected list item"));
}

ctx.consumeSpaces();

const auto tagPos = ctx.columnIdx;
const auto itemIndent = ctx.columnIdx;

const auto maybeLocalTag = parseTag(ctx);
if (!maybeLocalTag.has_value()) {
return std::unexpected(maybeLocalTag.error());
Expand All @@ -1060,7 +1057,7 @@ inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::st
switch (peekedType) {
case ValueType::List: {
if (!typeTag.empty()) {
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for list containing lists", tagPos));
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for list containing lists", itemIndent));
}
auto parsedValue = parseList(ctx, tag, static_cast<int>(line_indent));
if (!parsedValue.has_value()) {
Expand All @@ -1071,7 +1068,7 @@ inline std::expected<pmtv::pmt, ParseError> parseList(ParseContext& ctx, std::st
}
case ValueType::Map: {
if (!localTag.empty()) {
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for maps", tagPos));
return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for maps", itemIndent));
}
auto parsedValue = parseMap(ctx, static_cast<int>(line_indent));
if (!parsedValue.has_value()) {
Expand Down
65 changes: 64 additions & 1 deletion core/test/qa_YamlPmt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ void testYAML(std::string_view src, const pmtv::map_t expected) {
} else {
fmt::println(std::cerr, "Unexpected: {}", formatResult(deserializedMap));
expect(false);
return;
}

// Then test that serializing and deserializing the map again results in the same map
Expand All @@ -141,20 +142,32 @@ const boost::ut::suite YamlPmtTests = [] {
using namespace std::string_literals;
using namespace std::string_view_literals;

"Comments"_test = [] {
"Comments and Whitespace"_test = [] {
constexpr std::string_view src1 = R"(# Comment
double: !!float64 42 # Comment
string: "#Hello" # Comment
null: # Comment
#Comment: 43
# string: | # Comment
# Hello
number:
42
list:
[]
list2: [ 42
]
map:
{}
)";

pmtv::map_t expected;
expected["double"] = 42.0;
expected["string"] = "#Hello";
expected["null"] = std::monostate{};
expected["number"] = static_cast<int64_t>(42);
expected["list"] = std::vector<pmtv::pmt>{};
expected["list2"] = std::vector<pmtv::pmt>{static_cast<int64_t>(42)};
expected["map"] = pmtv::map_t{};

testYAML(src1, expected);
};
Expand Down Expand Up @@ -458,6 +471,8 @@ nullVector: !!null
- null
emptyVector: !!str []
emptyPmtVector: []
emptyAfterNewline:
[]
flowDouble: !!float64 [1, 2, 3]
flowString: !!str ["Hello, ", "World", "Multiple\nlines"]
flowMultiline: !!str [ "Hello, " , #]
Expand All @@ -478,6 +493,14 @@ flowMultiline: !!str [ "Hello, " , #]
- !!str [3, 4]
- { key: !!str [5, 6] }
nestedFlow: [ !!str [1, 2], [3, 4] ]
vectorWithBlockMap:
- name: ArraySink<double>
id: ArraySink
parameters:
name: Block
vectorWithColons:
- "key: value"
- "key2: value2"
)";

pmtv::map_t expected;
Expand All @@ -492,12 +515,15 @@ nestedFlow: [ !!str [1, 2], [3, 4] ]
expected["nullVector"] = std::monostate{};
expected["emptyVector"] = std::vector<std::string>{};
expected["emptyPmtVector"] = std::vector<pmtv::pmt>{};
expected["emptyAfterNewline"] = std::vector<pmtv::pmt>{};
expected["flowDouble"] = std::vector<double>{1.0, 2.0, 3.0};
expected["flowString"] = std::vector<std::string>{"Hello, ", "World", "Multiple\nlines"};
expected["flowMultiline"] = std::vector<std::string>{"Hello, ", "][", "World", "Multiple\nlines"};
expected["nestedVector"] = std::vector<pmtv::pmt>{std::vector<std::string>{"1", "2"}, std::vector<pmtv::pmt>{static_cast<int64_t>(3), static_cast<int64_t>(4)}};
expected["nestedFlow"] = std::vector<pmtv::pmt>{std::vector<std::string>{"1", "2"}, std::vector<pmtv::pmt>{static_cast<int64_t>(3), static_cast<int64_t>(4)}};
expected["nestedVector2"] = std::vector<pmtv::pmt>{static_cast<int64_t>(42), std::vector<std::string>{"1", "2"}, std::vector<std::string>{"3", "4"}, pmtv::map_t{{"key", std::vector<std::string>{"5", "6"}}}};
expected["vectorWithBlockMap"] = std::vector<pmtv::pmt>{pmtv::map_t{{"name", "ArraySink<double>"}, {"id", "ArraySink"}, {"parameters", pmtv::map_t{{"name", "Block"}}}}};
expected["vectorWithColons"] = std::vector<pmtv::pmt>{"key: value", "key2: value2"};

testYAML(src1, expected);

Expand Down Expand Up @@ -623,6 +649,43 @@ key#Comment: foo
expect(eq(formatResult(yaml::deserialize(invalidKeyComment)), "Error in 3:1: Could not find key/value separator ':'"sv));
};

"GRC"_test = [] {
constexpr std::string_view src = R"(
blocks:
- name: ArraySink<double>
id: ArraySink
parameters:
name: ArraySink<double>
- name: ArraySource<double>
id: ArraySource
parameters:
name: ArraySource<double>
connections:
- [ArraySource<double>, [0, 0], ArraySink<double>, [1, 1]]
- [ArraySource<double>, [0, 1], ArraySink<double>, [1, 0]]
- [ArraySource<double>, [1, 0], ArraySink<double>, [0, 0]]
- [ArraySource<double>, [1, 1], ArraySink<double>, [0, 1]]
)";

pmtv::map_t expected;
pmtv::map_t block1;
block1["name"] = "ArraySink<double>";
block1["id"] = "ArraySink";
block1["parameters"] = pmtv::map_t{{"name", "ArraySink<double>"}};
pmtv::map_t block2;
block2["name"] = "ArraySource<double>";
block2["id"] = "ArraySource";
block2["parameters"] = pmtv::map_t{{"name", "ArraySource<double>"}};
expected["blocks"] = std::vector<pmtv::pmt>{block1, block2};
const auto zero = pmtv::pmt{static_cast<int64_t>(0)};
const auto one = pmtv::pmt{static_cast<int64_t>(1)};
using PmtVec = std::vector<pmtv::pmt>;

expected["connections"] = std::vector<pmtv::pmt>{PmtVec{"ArraySource<double>", std::vector{zero, zero}, "ArraySink<double>", std::vector{one, one}}, PmtVec{"ArraySource<double>", std::vector{zero, one}, "ArraySink<double>", std::vector{one, zero}}, PmtVec{"ArraySource<double>", std::vector{one, zero}, "ArraySink<double>", std::vector{zero, zero}}, PmtVec{"ArraySource<double>", std::vector{one, one}, "ArraySink<double>", std::vector{zero, one}}};

testYAML(src, expected);
};

"Complex"_test = [] {
constexpr std::string_view src = R"(
complex: !!complex64 (1.0, -1.0)
Expand Down

0 comments on commit e4597a6

Please sign in to comment.