fix(elasticsearch): Encode bulk action parameters as JSON

[jszwedko]

They are currently using string templating which, if there are special characters in the value, will
end up creating an invalid JSON payload; an issue that can be difficult to track down. This happened
in #21288.

Signed-off-by: Jesse Szwedko [email protected]

[datadog-vectordotdev]

Datadog Report

Branch report: jszwedko/fix-elasticsearch-message-encoding
Commit report: baf5930
Test service: vector

❌ 9 Failed (0 Known Flaky), 2193 Passed, 0 Skipped, 1m 31.93s Total Time

❌ Failed Tests (9)

This report shows up to 5 failed tests.

• sinks::elasticsearch::tests::allows_using_except_fields - vector - Details

  Expand for error

   thread 'sinks::elasticsearch::tests::allows_using_except_fields' panicked at src/sinks/elasticsearch/tests.rs:494:5:
   assertion \`left == right\` failed
     left: "{\"index\":{\"_type\":\"_doc\",\"_index\":\"purple\"}}\n{\"foo\":\"bar\",\"message\":\"hello there\"}\n"
    right: "{\"index\":{\"_index\":\"purple\",\"_type\":\"_doc\"}}\n{\"foo\":\"bar\",\"message\":\"hello there\"}\n"
   stack backtrace:
      0:     0x555e0fe60835 - std::backtrace_rs::backtrace::libunwind::trace::h1a07e5dba0da0cd2
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/libunwind.rs:105:5
      1:     0x555e0fe60835 - std::backtrace_rs::backtrace::trace_unsynchronized::h61b9b8394328c0bc
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
      2:     0x555e0fe60835 - std::sys_common::backtrace::_print_fmt::h1c5e18b460934cff
   ...
  

• sinks::elasticsearch::tests::allows_using_only_fields - vector - Details

  Expand for error

   thread 'sinks::elasticsearch::tests::allows_using_only_fields' panicked at src/sinks/elasticsearch/tests.rs:529:5:
   assertion \`left == right\` failed
     left: "{\"index\":{\"_type\":\"_doc\",\"_index\":\"purple\"}}\n{\"foo\":\"bar\"}\n"
    right: "{\"index\":{\"_index\":\"purple\",\"_type\":\"_doc\"}}\n{\"foo\":\"bar\"}\n"
   stack backtrace:
      0:     0x556d742ff835 - std::backtrace_rs::backtrace::libunwind::trace::h1a07e5dba0da0cd2
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/libunwind.rs:105:5
      1:     0x556d742ff835 - std::backtrace_rs::backtrace::trace_unsynchronized::h61b9b8394328c0bc
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
      2:     0x556d742ff835 - std::sys_common::backtrace::_print_fmt::h1c5e18b460934cff
   ...
  

• sinks::elasticsearch::tests::encode_datastream_mode - vector - Details

  Expand for error

   thread 'sinks::elasticsearch::tests::encode_datastream_mode' panicked at src/sinks/elasticsearch/tests.rs:251:5:
   assertion \`left == right\` failed
     left: "{\"create\":{\"_type\":\"_doc\",\"_index\":\"synthetics-testing-default\"}}\n{\"@timestamp\":\"2020-12-01T01:02:03Z\",\"data_stream\":{\"dataset\":\"testing\",\"namespace\":\"default\",\"type\":\"synthetics\"},\"message\":\"hello there\"}\n"
    right: "{\"create\":{\"_index\":\"synthetics-testing-default\",\"_type\":\"_doc\"}}\n{\"@timestamp\":\"2020-12-01T01:02:03Z\",\"data_stream\":{\"dataset\":\"testing\",\"namespace\":\"default\",\"type\":\"synthetics\"},\"message\":\"hello there\"}\n"
   stack backtrace:
      0:     0x5594ce0c1835 - std::backtrace_rs::backtrace::libunwind::trace::h1a07e5dba0da0cd2
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/libunwind.rs:105:5
      1:     0x5594ce0c1835 - std::backtrace_rs::backtrace::trace_unsynchronized::h61b9b8394328c0bc
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
      2:     0x5594ce0c1835 - std::sys_common::backtrace::_print_fmt::h1c5e18b460934cff
   ...
  

• sinks::elasticsearch::tests::encode_datastream_mode_no_routing - vector - Details

  Expand for error

   thread 'sinks::elasticsearch::tests::encode_datastream_mode_no_routing' panicked at src/sinks/elasticsearch/tests.rs:306:5:
   assertion \`left == right\` failed
     left: "{\"create\":{\"_type\":\"_doc\",\"_index\":\"logs-generic-something\"}}\n{\"@timestamp\":\"2020-12-01T01:02:03Z\",\"data_stream\":{\"dataset\":\"testing\",\"namespace\":\"something\",\"type\":\"synthetics\"},\"message\":\"hello there\"}\n"
    right: "{\"create\":{\"_index\":\"logs-generic-something\",\"_type\":\"_doc\"}}\n{\"@timestamp\":\"2020-12-01T01:02:03Z\",\"data_stream\":{\"dataset\":\"testing\",\"namespace\":\"something\",\"type\":\"synthetics\"},\"message\":\"hello there\"}\n"
   stack backtrace:
      0:     0x55a5b8510835 - std::backtrace_rs::backtrace::libunwind::trace::h1a07e5dba0da0cd2
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/libunwind.rs:105:5
      1:     0x55a5b8510835 - std::backtrace_rs::backtrace::trace_unsynchronized::h61b9b8394328c0bc
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
      2:     0x55a5b8510835 - std::sys_common::backtrace::_print_fmt::h1c5e18b460934cff
   ...
  

• sinks::elasticsearch::tests::encode_datastream_mode_no_sync - vector - Details

  Expand for error

   thread 'sinks::elasticsearch::tests::encode_datastream_mode_no_sync' panicked at src/sinks/elasticsearch/tests.rs:458:5:
   assertion \`left == right\` failed
     left: "{\"create\":{\"_type\":\"_doc\",\"_index\":\"synthetics-testing-something\"}}\n{\"@timestamp\":\"2020-12-01T01:02:03Z\",\"data_stream\":{\"dataset\":\"testing\",\"type\":\"synthetics\"},\"message\":\"hello there\"}\n"
    right: "{\"create\":{\"_index\":\"synthetics-testing-something\",\"_type\":\"_doc\"}}\n{\"@timestamp\":\"2020-12-01T01:02:03Z\",\"data_stream\":{\"dataset\":\"testing\",\"type\":\"synthetics\"},\"message\":\"hello there\"}\n"
   stack backtrace:
      0:     0x561f0f5e4835 - std::backtrace_rs::backtrace::libunwind::trace::h1a07e5dba0da0cd2
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/libunwind.rs:105:5
      1:     0x561f0f5e4835 - std::backtrace_rs::backtrace::trace_unsynchronized::h61b9b8394328c0bc
                                  at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
      2:     0x561f0f5e4835 - std::sys_common::backtrace::_print_fmt::h1c5e18b460934cff
   ...
  

[pront]

Should we add an .unwrap() here?

[jszwedko]

Mmm, yeah, I just used the pattern the other tests here are using, but yes, it would make sense to use unwrap on all of these