From f77e199389fac4e6c48a71ccd672012cf49f2db7 Mon Sep 17 00:00:00 2001 From: Frank Osterfeld Date: Fri, 22 Nov 2024 10:40:19 +0100 Subject: [PATCH] Add support for flow style maps and lists Signed-off-by: Frank Osterfeld --- core/include/gnuradio-4.0/YamlPmt.hpp | 302 ++++++++++++++++++-------- core/test/qa_YamlPmt.cpp | 51 +++++ 2 files changed, 267 insertions(+), 86 deletions(-) diff --git a/core/include/gnuradio-4.0/YamlPmt.hpp b/core/include/gnuradio-4.0/YamlPmt.hpp index 678c6077c..614b2ac42 100644 --- a/core/include/gnuradio-4.0/YamlPmt.hpp +++ b/core/include/gnuradio-4.0/YamlPmt.hpp @@ -213,11 +213,37 @@ struct ParseContext { std::size_t lineIdx = 0; std::size_t columnIdx = 0; - bool hasMoreLines() const { return lineIdx < lines.size(); } - bool documentStart() const { return lineIdx == 0 && columnIdx == 0; } - bool startsWith(std::string_view sv) const { return lines[lineIdx].substr(columnIdx).starts_with(sv); } + bool startsWith(std::string_view sv) const { + if (atEndOfLine()) { + return false; + } + return lines[lineIdx].substr(columnIdx).starts_with(sv); + } + + bool startsWith(char c) const { + if (atEndOfLine()) { + return false; + } + return lines[lineIdx][columnIdx] == c; + } + + bool consumeIfStartsWith(std::string_view sv) { + if (startsWith(sv)) { + consume(sv.size()); + return true; + } + return false; + } + + bool consumeIfStartsWith(char c) { + if (startsWith(c)) { + consume(1); + return true; + } + return false; + } char front() const { return lines[lineIdx][columnIdx]; } @@ -245,9 +271,19 @@ struct ParseContext { } } + void consumeWhitespaceAndComments() { + while (!atEndOfDocument()) { + consumeSpaces(); + if (!atEndOfLine() && front() != '#') { + break; + } + skipToNextLine(); + } + } + std::string_view remainingLine() const { return atEndOfLine() ? std::string_view{} : lines[lineIdx].substr(columnIdx); } - bool atEndOfLine() const { return columnIdx == lines[lineIdx].size(); } + bool atEndOfLine() const { return atEndOfDocument() || columnIdx == lines[lineIdx].size(); } bool atEndOfDocument() const { return lineIdx == lines.size(); } @@ -535,13 +571,18 @@ inline size_t findClosingQuote(std::string_view sv, char quoteChar) { return std::string_view::npos; } -inline std::pair findString(std::string_view sv) { +inline std::pair findString(std::string_view sv, std::string_view extraDelimiters = {}) { if (sv.size() < 2) { return {0, sv.size()}; } const char firstChar = sv.front(); const auto quoted = firstChar == '"' || firstChar == '\''; if (!quoted) { + // Check for extra delimiter first (',' for flow, ':' for keys) + auto delimPos = sv.find_first_of(extraDelimiters); + if (delimPos != std::string_view::npos) { + return {0, delimPos}; + } // Ignore trailing comments auto commentPos = sv.find('#'); if (commentPos != std::string_view::npos) { @@ -558,8 +599,8 @@ inline std::pair findString(std::string_view sv) { } template -inline std::expected parseNextString(ParseContext& ctx, Fnc fnc) { - auto [offset, length] = findString(ctx.remainingLine()); +inline std::expected parseNextString(ParseContext& ctx, std::string_view extraDelimiters, Fnc fnc) { + auto [offset, length] = findString(ctx.remainingLine(), extraDelimiters); ctx.consume(offset); const auto fncResult = fnc(offset, ctx.remainingLine().substr(0, length)); if (!fncResult) { @@ -570,6 +611,45 @@ inline std::expected parseNextString(ParseContext& ctx, F return *fncResult; } +inline std::expected parsePlainScalar(ParseContext& ctx, std::string_view typeTag, std::string_view extraDelimiters = {}) { + // if we have a type tag, enforce the type + if (!typeTag.empty()) { + return parseNextString(ctx, extraDelimiters, [typeTag](std::size_t, std::string_view sv) { return applyTag, ParseAs>(typeTag, sv); }); + } + + // fallback for parsing without a YAML tag + return parseNextString(ctx, extraDelimiters, [&](std::size_t quoteOffset, std::string_view sv) -> std::expected { + // If it's quoted, treat as string + if (quoteOffset > 0) { + return resolveYamlEscapes_quoted(sv); + } + + // null + if (sv.empty() || sv == "null" || sv == "Null" || sv == "NULL" || sv == "~") { + return std::monostate{}; + } + + // boolean + if (sv == "true" || sv == "True" || sv == "TRUE") { + return true; + } + if (sv == "false" || sv == "False" || sv == "FALSE") { + return false; + } + + // try numbers + if (const auto asInt = parseAs(sv)) { + return *asInt; + } + if (const auto asDouble = parseAs(sv)) { + return *asDouble; + } + + // Anything else: string + return parseAs(sv).transform_error([&](ValueParseError error) { return ValueParseError{quoteOffset + error.offset, error.message}; }); + }); +} + inline std::expected parseScalar(ParseContext& ctx, std::string_view typeTag, int currentIndentLevel) { // remove leading spaces ctx.consumeSpaces(); @@ -579,11 +659,7 @@ inline std::expected parseScalar(ParseContext& ctx, std:: char indicator = ctx.front(); ctx.consume(1); - bool trailingNewline = true; - if (ctx.startsWith("-")) { - trailingNewline = false; - ctx.consume(1); - } + const auto trailingNewline = !ctx.consumeIfStartsWith('-'); ctx.consumeSpaces(); const auto& [_, length] = findString(ctx.remainingLine()); @@ -596,7 +672,7 @@ inline std::expected parseScalar(ParseContext& ctx, std:: bool firstLine = true; ctx.skipToNextLine(); - for (; ctx.hasMoreLines(); ctx.skipToNextLine()) { + for (; !ctx.atEndOfDocument(); ctx.skipToNextLine()) { auto lineIndent = ctx.currentIndent(); if (lineIndent == std::string_view::npos) { // empty or whitespace-only line @@ -647,44 +723,7 @@ inline std::expected parseScalar(ParseContext& ctx, std:: return result; } - const auto result = [&] -> std::expected { - // if we have a type tag, enforce the type - if (!typeTag.empty()) { - return parseNextString(ctx, [typeTag](std::size_t, std::string_view sv) { return applyTag, ParseAs>(typeTag, sv); }); - } - - // fallback for parsing without a YAML tag - return parseNextString(ctx, [&](std::size_t quoteOffset, std::string_view sv) -> std::expected { - // If it's quoted, treat as string - if (quoteOffset > 0) { - return resolveYamlEscapes_quoted(sv); - } - - // null - if (sv.empty() || sv == "null" || sv == "Null" || sv == "NULL" || sv == "~") { - return std::monostate{}; - } - - // boolean - if (sv == "true" || sv == "True" || sv == "TRUE") { - return true; - } - if (sv == "false" || sv == "False" || sv == "FALSE") { - return false; - } - - // try numbers - if (const auto asInt = parseAs(sv)) { - return *asInt; - } - if (const auto asDouble = parseAs(sv)) { - return *asDouble; - } - - // Anything else: string - return parseAs(sv).transform_error([&](ValueParseError error) { return ValueParseError{quoteOffset + error.offset, error.message}; }); - }); - }(); + const auto result = parsePlainScalar(ctx, typeTag); if (!result) { return std::unexpected(result.error()); @@ -717,7 +756,7 @@ inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) { return ValueType::Scalar; } ctx.skipToNextLine(); - while (ctx.hasMoreLines()) { + while (!ctx.atEndOfDocument()) { if (ctx.startsWith("#")) { ctx.skipToNextLine(); continue; @@ -742,14 +781,14 @@ inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) { return ValueType::Scalar; } -inline std::expected parseKey(ParseContext& ctx) { +inline std::expected parseKey(ParseContext& ctx, std::string_view extraDelimiters = {}) { ctx.consumeSpaces(); if (ctx.startsWith("-")) { return std::unexpected(ctx.makeError("Unexpected list item in map.")); } - const auto& [quoteOffset, length] = findString(ctx.remainingLine()); + const auto& [quoteOffset, length] = findString(ctx.remainingLine(), extraDelimiters); if (quoteOffset > 0) { // quoted auto maybeKey = resolveYamlEscapes_quoted(ctx.remainingLine().substr(quoteOffset, length)); @@ -777,19 +816,125 @@ inline std::expected parseKey(ParseContext& ctx) { return key; } -inline std::expected parseMap(ParseContext& ctx, int parentIndentLevel) { - pmtv::map_t map; +template +struct ConvertList { + pmtv::pmt operator()(const std::vector& list) { + if constexpr (std::is_same_v) { + return std::monostate{}; + } else { + auto resultView = list | std::views::transform([](const auto& item) { return std::get(item); }); + return std::vector(resultView.begin(), resultView.end()); + } + } +}; + +enum class FlowType { List, Map }; +template +inline auto parseFlow(ParseContext& ctx, std::string_view typeTag, int parentIndentLevel) { + using ResultType = std::conditional_t; + using TemporaryResultType = std::conditional_t, pmtv::map_t>; + using ReturnType = std::expected; + auto makeError = [&](std::string message) -> ReturnType { return std::unexpected(ctx.makeError(std::move(message))); }; + const auto startLineIdx = ctx.lineIdx; + + constexpr auto closingChar = Type == FlowType::List ? ']' : '}'; + + TemporaryResultType result; + + while (!ctx.atEndOfDocument()) { + ctx.consumeWhitespaceAndComments(); + + if (ctx.atEndOfDocument()) { + return makeError("Unexpected end of document"); + } + if (ctx.consumeIfStartsWith(closingChar)) { + // end of flow sequence + break; + } + if (ctx.lineIdx > startLineIdx && parentIndentLevel >= 0 && ctx.currentIndent() <= static_cast(parentIndentLevel)) { + return makeError("Flow sequence insufficiently indented"); + } + + auto parseElementValue = [&] -> std::expected { + const auto maybeTag = parseTag(ctx); + if (!maybeTag.has_value()) { + return ReturnType{std::unexpected(maybeTag.error())}; + } + auto nestedTag = maybeTag.value(); + + if (!typeTag.empty() && !nestedTag.empty()) { + return makeError("Cannot have type tag for both list and list item"); + } + + const auto localTag = !nestedTag.empty() ? nestedTag : typeTag; + + ctx.consumeWhitespaceAndComments(); + + if (ctx.consumeIfStartsWith('[')) { + return parseFlow(ctx, localTag, parentIndentLevel); + } + if (ctx.consumeIfStartsWith('{')) { + return parseFlow(ctx, localTag, parentIndentLevel); + } + + constexpr std::string_view extraDelimiters = Type == FlowType::List ? ",]" : ",}"; + return parsePlainScalar(ctx, localTag, extraDelimiters); + }; + + if constexpr (Type == FlowType::List) { + auto value = parseElementValue(); + if (!value.has_value()) { + return ReturnType{std::unexpected(value.error())}; + } + result.push_back(std::move(value.value())); + } else { + auto key = parseKey(ctx, ","); + if (!key.has_value()) { + return ReturnType{std::unexpected(key.error())}; + } + ctx.consumeWhitespaceAndComments(); + auto value = parseElementValue(); + if (!value.has_value()) { + return ReturnType{std::unexpected(value.error())}; + } + result[std::move(key.value())] = std::move(value.value()); + } + ctx.consumeWhitespaceAndComments(); + if (ctx.consumeIfStartsWith(",")) { + // continue to next value + } else if (ctx.consumeIfStartsWith(closingChar)) { + // end of flow sequence + break; + } else { + if constexpr (Type == FlowType::List) { + return makeError("Expected ',' or ']'"); + } else { + return makeError("Expected ',' or '}'"); + } + } + } + if constexpr (Type == FlowType::List) { + if (typeTag.empty()) { + return ReturnType{result}; + } + return ReturnType{applyTag(typeTag, result)}; + } else { + return ReturnType{result}; + } +} +inline std::expected parseMap(ParseContext& ctx, int parentIndentLevel) { if (!ctx.documentStart()) { - if (ctx.remainingLine() == "{}") { - // TODO support flow style for non-empty maps + if (ctx.consumeIfStartsWith("{")) { + auto map = parseFlow(ctx, "", parentIndentLevel); ctx.skipToNextLine(); return map; } - ctx.skipToNextLine(); } - while (ctx.hasMoreLines()) { + pmtv::map_t map; + + while (!ctx.atEndOfDocument()) { ctx.consumeSpaces(); if (ctx.atEndOfLine() || ctx.startsWith("#")) { // skip empty lines and comments @@ -854,33 +999,18 @@ inline std::expected parseMap(ParseContext& ctx, int pa return map; } -template -struct ConvertList { - pmtv::pmt operator()(const std::vector& list) { - if constexpr (std::is_same_v) { - return std::monostate{}; - } else { - auto resultView = list | std::views::transform([](const auto& item) { return std::get(item); }); - return std::vector(resultView.begin(), resultView.end()); - } - } -}; - inline std::expected parseList(ParseContext& ctx, std::string_view typeTag, int parentIndentLevel) { - std::vector list; - if (ctx.remainingLine() == "[]") { - // TODO support flow style for non-empty lists + if (ctx.consumeIfStartsWith("[")) { + auto l = parseFlow(ctx, typeTag, parentIndentLevel); ctx.skipToNextLine(); - - if (typeTag.empty()) { - return list; - } - return applyTag(typeTag, list); + return l; } + std::vector list; + ctx.skipToNextLine(); - while (ctx.hasMoreLines()) { + while (!ctx.atEndOfDocument()) { ctx.consumeSpaces(); if (ctx.atEndOfLine() || ctx.startsWith("#")) { // skip empty lines and comments @@ -896,13 +1026,11 @@ inline std::expected parseList(ParseContext& ctx, std::st ctx.consumeSpaces(); - if (!ctx.remainingLine().starts_with('-')) { + if (!ctx.consumeIfStartsWith('-')) { // not a list item return std::unexpected(ctx.makeError("Expected list item")); } - // remove leading '- ' from the line - ctx.consume(1); ctx.consumeSpaces(); const auto maybeLocalTag = parseTag(ctx); @@ -914,6 +1042,8 @@ inline std::expected parseList(ParseContext& ctx, std::st return std::unexpected(ctx.makeError("Cannot have type tag for both list and list item")); } + const auto tag = !typeTag.empty() ? typeTag : localTag; + ctx.consumeSpaces(); const auto peekedType = peekToFindValueType(ctx, static_cast(line_indent)); @@ -922,7 +1052,7 @@ inline std::expected parseList(ParseContext& ctx, std::st if (!typeTag.empty()) { return std::unexpected(ctx.makeError("Cannot have type tag for list containing lists")); } - auto parsedValue = parseList(ctx, "", static_cast(line_indent)); + auto parsedValue = parseList(ctx, tag, static_cast(line_indent)); if (!parsedValue.has_value()) { return std::unexpected(parsedValue.error()); } @@ -941,7 +1071,7 @@ inline std::expected parseList(ParseContext& ctx, std::st break; } case ValueType::Scalar: { - auto parsedValue = parseScalar(ctx, !typeTag.empty() ? typeTag : localTag, static_cast(line_indent)); + auto parsedValue = parseScalar(ctx, tag, static_cast(line_indent)); if (!parsedValue.has_value()) { return std::unexpected(parsedValue.error()); } diff --git a/core/test/qa_YamlPmt.cpp b/core/test/qa_YamlPmt.cpp index d0af92ab9..e5e58e8f8 100644 --- a/core/test/qa_YamlPmt.cpp +++ b/core/test/qa_YamlPmt.cpp @@ -419,6 +419,21 @@ complexVector: !!complex64 - (3.0, -3.0) emptyVector: !!str [] emptyPmtVector: [] +flowDouble: !!float64 [1, 2, 3] +flowString: !!str ["Hello, ", "World", "Multiple\nlines"] +flowMultiline: !!str [ "Hello, " , #] + "][", # Comment , + "World" , + "Multiple\nlines" +] +nestedVector: + - !!str + - 1 + - 2 + - + - 3 + - 4 +nestedFlow: [ !!str [1, 2], [3, 4] ] )"; pmtv::map_t expected; @@ -432,10 +447,46 @@ emptyPmtVector: [] expected["complexVector"] = std::vector>{{1.0, -1.0}, {2.0, -2.0}, {3.0, -3.0}}; expected["emptyVector"] = std::vector{}; expected["emptyPmtVector"] = std::vector{}; + expected["flowDouble"] = std::vector{1.0, 2.0, 3.0}; + expected["flowString"] = std::vector{"Hello, ", "World", "Multiple\nlines"}; + expected["flowMultiline"] = std::vector{"Hello, ", "][", "World", "Multiple\nlines"}; + expected["nestedVector"] = std::vector{std::vector{"1", "2"}, std::vector{static_cast(3), static_cast(4)}}; + expected["nestedFlow"] = std::vector{std::vector{"1", "2"}, std::vector{static_cast(3), static_cast(4)}}; testYAML(src1, expected); }; + "Maps"_test = [] { + constexpr std::string_view src = R"yaml( +simple: + key1: !!int8 42 + key2: !!int8 43 +empty: {} +nested: + key1: + key2: !!int8 42 + key3: !!int8 43 + key4: + key5: !!int8 44 + key6: !!int8 45 +flow: {key1: !!int8 42, key2: !!int8 43} +flow_multiline: {key1: !!int8 42, + key2: !!int8 43} +flow_nested: {key1: {key2: !!int8 42, key3: !!int8 43}, key4: {key5: !!int8 44, key6: !!int8 45}} +flow_braces: {"}{": !!int8 42} +)yaml"; + + pmtv::map_t expected; + expected["simple"] = pmtv::map_t{{"key1", static_cast(42)}, {"key2", static_cast(43)}}; + expected["empty"] = pmtv::map_t{}; + expected["nested"] = pmtv::map_t{{"key1", pmtv::map_t{{"key2", static_cast(42)}, {"key3", static_cast(43)}}}, {"key4", pmtv::map_t{{"key5", static_cast(44)}, {"key6", static_cast(45)}}}}; + expected["flow"] = pmtv::map_t{{"key1", static_cast(42)}, {"key2", static_cast(43)}}; + expected["flow_multiline"] = pmtv::map_t{{"key1", static_cast(42)}, {"key2", static_cast(43)}}; + expected["flow_nested"] = pmtv::map_t{{"key1", pmtv::map_t{{"key2", static_cast(42)}, {"key3", static_cast(43)}}}, {"key4", pmtv::map_t{{"key5", static_cast(44)}, {"key6", static_cast(45)}}}}; + expected["flow_braces"] = pmtv::map_t{{"}{", static_cast(42)}}; + testYAML(src, expected); + }; + "Complex"_test = [] { constexpr std::string_view src = R"( complex: !!complex64 (1.0, -1.0)