Handle newlines in strings (LLNL#506)

python/caliper-reader/caliperreader/caliperstreamreader.py

@@ -142,7 +142,10 @@ def _read_cali_record(line):
     for c in iterator:
         if c == '\\':
             c = next(iterator)
-            string += c
+            if c == 'n':
+                string += '\n'
+            else:
+                string += c
         elif c == ',':
             entry.append(string)
             result[entry[0]] = entry[1:]

src/common/util/format_util.h

@@ -15,25 +15,55 @@ namespace util
 /// \brief Write string \a str to \a os,
 ///   escaping all characters in \a mask_chars with \a esc.
 inline std::ostream&
-write_esc_string(std::ostream& os, const char* str, std::string::size_type size, const char* mask_chars = "\\\"", char esc = '\\')
+write_json_esc_string(std::ostream& os, const char* str, std::string::size_type size)
 {
     for (size_t i = 0; i < size; ++i) {
-        for (const char* p = mask_chars; *p; ++p)
-            if (str[i] == *p) {
-                os << esc;
-                break;
-            }
-
-        os << str[i];
+        const char c = str[i];
+
+        if (c == '\n') // handle newline in string
+            os << "\\n";
+        if (c < 0x20)  // skip control characters
+            continue;
+        if (c == '\\' || c == '\"')
+            os << '\\';
+
+        os << c;
     }
-    
+
     return os;
 }
 
+/// \brief Write string \a str to \a os,
+///   escaping all characters in \a mask_chars with \a esc.
+inline std::ostream&
+write_cali_esc_string(std::ostream& os, const char* str, std::string::size_type size)
+{
+    for (size_t i = 0; i < size; ++i) {
+        const char c = str[i];
+
+        if (c == '\n') // handle newline in string
+            os << "\\n";
+        if (c < 0x20)  // skip control characters
+            continue;
+        if (c == '\\' || c == ',' || c == '=')
+            os << '\\';
+
+        os << c;
+    }
+
+    return os;
+}
+
+inline std::ostream&
+write_json_esc_string(std::ostream& os, const std::string& str)
+{
+    return write_json_esc_string(os, str.data(), str.size());
+}
+
 inline std::ostream&
-write_esc_string(std::ostream& os, const std::string& str, const char* mask_chars = "\\\"", char esc = '\\')
+write_cali_esc_string(std::ostream& os, const std::string& str)
 {
-    return write_esc_string(os, str.data(), str.size(), mask_chars, esc);
+    return write_cali_esc_string(os, str.data(), str.size());
 }
 
 std::ostream&

src/reader/CaliReader.cpp

@@ -38,6 +38,9 @@ vector<string> split(const string& line, char sep, bool keep_escape = false) {
             vec.emplace_back(std::move(str));
             str.clear();
             str.reserve(line.size());
+        } else if (escaped && !keep_escape && *it == 'n') {
+            str.push_back('\n');
+            escaped = false;
         } else {
             str.push_back(*it);
             escaped = false;

src/reader/CaliWriter.cpp

@@ -19,14 +19,12 @@ using namespace cali;
 namespace
 {
 
-const char* esc_chars { "\\,=\n" }; ///< characters that need to be escaped
-
 void write_node_content(std::ostream& os, const cali::Node* node)
 {
     os << "__rec=node,id=" << node->id()
        << ",attr="         << node->attribute();
 
-    util::write_esc_string(os << ",data=", node->data().to_string(), esc_chars);
+    util::write_cali_esc_string(os << ",data=", node->data().to_string());
 
     if (node->parent() && node->parent()->id() != CALI_INV_ID)
         os << ",parent=" << node->parent()->id();
@@ -36,12 +34,12 @@ void write_node_content(std::ostream& os, const cali::Node* node)
 
 void write_record_content(std::ostream& os, const char* record_type, int nr, int ni, const std::vector<Entry>& rec) {
     os << "__rec=" << record_type;
-            
+
     // write reference entries
 
     if (nr > 0) {
         os << ",ref";
-            
+
         for (const Entry& e : rec)
             if (e.is_reference())
                 os << '=' << e.node()->id();
@@ -60,7 +58,7 @@ void write_record_content(std::ostream& os, const char* record_type, int nr, int
 
         for (const Entry& e : rec)
             if (e.is_immediate())
-                util::write_esc_string(os << '=', e.value().to_string(), esc_chars);
+                util::write_cali_esc_string(os << '=', e.value().to_string());
     }
 
     os << '\n';
@@ -83,7 +81,7 @@ struct CaliWriter::CaliWriterImpl
         : m_os(os),
           m_num_written(0)
     { }
-    
+
     void recursive_write_node(const CaliperMetadataAccessInterface& db, cali_id_t id)
     {
         if (id < 11) // don't write the hard-coded metadata nodes
@@ -114,7 +112,7 @@ struct CaliWriter::CaliWriterImpl
                 g(m_os_lock);
 
             std::ostream* real_os = m_os.stream();
-            
+
             ::write_node_content(*real_os, node);
             ++m_num_written;
         }
@@ -125,7 +123,7 @@ struct CaliWriter::CaliWriterImpl
 
             if (m_written_nodes.count(id) > 0)
                 return;
-       
+
             m_written_nodes.insert(id);
         }
     }
@@ -135,10 +133,10 @@ struct CaliWriter::CaliWriterImpl
                          const std::vector<Entry>& rec)
     {
         // write node entries; count the number of ref and immediate entries
-        
+
         int nr = 0;
         int ni = 0;
-        
+
         for (const Entry& e : rec) {
             if (e.is_reference()) {
                 recursive_write_node(db, e.node()->id());
@@ -150,7 +148,7 @@ struct CaliWriter::CaliWriterImpl
         }
 
         // write the record
-        
+
         {
             std::lock_guard<std::mutex>
                 g(m_os_lock);

src/reader/JsonFormatter.cpp

@@ -194,14 +194,14 @@ struct JsonFormatter::JsonFormatterImpl
             for (auto &p : quote_kvs) {
                 *real_os << (count++ > 0 ? "," : "")
                          << (m_opt_pretty ? "\n\t" : "");
-                util::write_esc_string(*real_os << "\"", p.first) << "\":";
-                util::write_esc_string(*real_os << "\"", p.second) << "\"";
+                util::write_json_esc_string(*real_os << "\"", p.first) << "\":";
+                util::write_json_esc_string(*real_os << "\"", p.second) << "\"";
             }
             for (auto &p : noquote_kvs) {
                 *real_os << (count++ > 0 ? "," : "")
                          << (m_opt_pretty ? "\n\t" : "");
-                util::write_esc_string(*real_os << "\"", p.first) << "\":";
-                util::write_esc_string(*real_os, p.second);
+                util::write_json_esc_string(*real_os << "\"", p.first) << "\":";
+                util::write_json_esc_string(*real_os, p.second);
             }
 
             *real_os << (m_opt_pretty ? "\n" : "") << "}";
@@ -242,8 +242,8 @@ struct JsonFormatter::JsonFormatterImpl
 
             // print meta-info
             for (const Node* node = a.node(); node && node->attribute() != CALI_INV_ID; node = node->parent()) {
-                util::write_esc_string(os << ",\"", db.get_attribute(node->attribute()).name()) << "\": ";
-                util::write_esc_string(os << "\"", node->data().to_string()) << '\"';
+                util::write_json_esc_string(os << ",\"", db.get_attribute(node->attribute()).name()) << "\": ";
+                util::write_json_esc_string(os << "\"", node->data().to_string()) << '\"';
             }
 
             os << "}";
@@ -276,8 +276,8 @@ struct JsonFormatter::JsonFormatterImpl
             if (m_opt_pretty)
                 os << '\t';
 
-            util::write_esc_string(os << '\"', db.get_attribute(p.first).name()) << "\": ";
-            util::write_esc_string(os << '\"', p.second) << '\"';
+            util::write_json_esc_string(os << '\"', db.get_attribute(p.first).name()) << "\": ";
+            util::write_json_esc_string(os << '\"', p.second) << '\"';
         }
 
         return os;

src/reader/JsonSplitFormatter.cpp

@@ -51,8 +51,8 @@ class Hierarchy
         const std::string& label() const { return m_label; }
 
         std::ostream& write_json(std::ostream& os) const {
-            util::write_esc_string(os << "{ \"label\": \"",  m_label ) << "\"";
-            util::write_esc_string(os << ", \"column\": \"", m_column) << "\"";
+            util::write_json_esc_string(os << "{ \"label\": \"",  m_label ) << "\"";
+            util::write_json_esc_string(os << ", \"column\": \"", m_column) << "\"";
 
             if (parent() && parent()->id() != CALI_INV_ID)
                 os << ", \"parent\": " << parent()->id();
@@ -250,7 +250,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
 
         if (!path.attributes.empty())
             columns.push_back(path);
-        
+
         return columns;
     }
 
@@ -281,7 +281,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
         for (const Entry& e : list)
             if (e.attribute() == attr.id()) {
                 if (quote)
-                    util::write_esc_string(os << "\"", e.value().to_string()) << "\"";
+                    util::write_json_esc_string(os << "\"", e.value().to_string()) << "\"";
                 else
                     os << e.value().to_string();
 
@@ -294,7 +294,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
     void process_record(const CaliperMetadataAccessInterface& db, const EntryList& list) {
         std::lock_guard<std::mutex>
             g(m_records_lock);
-        
+
         m_records.push_back(list);
     }
 
@@ -316,8 +316,8 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
                 global_vals[e.attribute()] = e.value().to_string();
 
         for (auto &p : global_vals) {
-            util::write_esc_string(os << ",\n  \"", db.get_attribute(p.first).name()) << "\": ";
-            util::write_esc_string(os << '"', p.second) << '\"';
+            util::write_json_esc_string(os << ",\n  \"", db.get_attribute(p.first).name()) << "\": ";
+            util::write_json_esc_string(os << '"', p.second) << '\"';
         }
 
         return os;
@@ -340,8 +340,8 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
                 if (attr.id() < 12 || attr.is_hidden())
                     continue;
 
-                util::write_esc_string(os << ", \"", attr.name_c_str())        << "\": ";
-                util::write_esc_string(os << "\"",   node->data().to_string()) << "\"";
+                util::write_json_esc_string(os << ", \"", attr.name_c_str())        << "\": ";
+                util::write_json_esc_string(os << "\"",   node->data().to_string()) << "\"";
             }
         }
 
@@ -355,7 +355,7 @@ struct JsonSplitFormatter::JsonSplitFormatterImpl
         {
             int count = 0;
             for (const Column& c : columns)
-                util::write_esc_string(os << (count++ > 0 ? ", " : " ") << "\"", c.title) << "\"";
+                util::write_json_esc_string(os << (count++ > 0 ? ", " : " ") << "\"", c.title) << "\"";
         }
 
         // close "columns", start "column_metadata"

test/ci_app_tests/ci_test_basic.cpp

@@ -2,7 +2,9 @@
 
 #include "caliper/cali.h"
 
-int main()
+#include <cstring>
+
+int main(int argc, char* argv[])
 {
     std::map<const char*, cali::Variant> metadata = {
         { "meta.int", cali::Variant(42) }
@@ -12,6 +14,9 @@ int main()
     cali_set_string_byname(" =\\weird \"\"attribute\"=  ", "  \\\\ weird,\" name\",");
     cali_set_global_string_byname(" =\\weird \"\" global attribute\"=  ", "  \\\\ weird,\" name\",");
 
+    if (argc > 1 && strcmp(argv[1], "newline") == 0)
+        cali_set_string_byname("newline", "A newline:\n!");
+
     cali::Annotation phase_ann("phase", metadata);
     std::size_t size = 8;
     cali::Annotation size_annot("dgs");

test/ci_app_tests/test_basictrace.py

@@ -152,7 +152,7 @@ def test_globals_selection(self):
             snapshots, { 'cali.caliper.version' } ) )
 
     def test_esc(self):
-        target_cmd = [ './ci_test_basic' ]
+        target_cmd = [ './ci_test_basic', 'newline' ]
         query_cmd  = [ '../../src/tools/cali-query/cali-query', '-j', '-s', 'cali.event.set' ]
 
         caliper_config = {
@@ -163,7 +163,10 @@ def test_esc(self):
 
         obj = json.loads( cat.run_test_with_query(target_cmd, query_cmd, caliper_config) )
 
+        self.assertEqual(len(obj), 2)
+
         self.assertEqual(obj[0]['event.set# =\\weird ""attribute"=  '], '  \\\\ weird," name",' )
+        self.assertEqual(obj[1]['event.set#newline'], 'A newline:\n!')
 
     def test_macros(self):
         # Use ConfigManager API here