From 0b80d1e37a1c9179e4b79ac384a9a1bd83d6708a Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 2 Sep 2024 12:12:30 +0200 Subject: [PATCH 1/5] [test] updated test on overflow of fixed 'decode_size_limit_bytes' to not raise an error bu create an Event, tag it appropriately and slice the offending data into 'message' field --- spec/codecs/json_lines_spec.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index 7fc9ff5..581eb5b 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -127,6 +127,13 @@ subject.decode(maximum_payload) }.not_to raise_error end + + it "should raise an error if the max bytes are exceeded" do + subject.decode(maximum_payload << "z") do |event| + expect(event.get("tags")).to include("_jsonparse_too_big_failure") + expect(event.get("message").size).to eq(decode_size_limit_bytes) + end + end it "should raise an error if the max bytes are exceeded" do expect { From 9a5fd1beb200a72a178950868055b16eafa3443e Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 2 Sep 2024 12:48:07 +0200 Subject: [PATCH 2/5] In case of buffer overflow create an event with a slice of the data and tag the event with an error --- lib/logstash/codecs/json_lines.rb | 7 +++++++ spec/codecs/json_lines_spec.rb | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index c0a9795..b985f3e 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -69,6 +69,13 @@ def decode(data, &block) @buffer.extract(data).each do |line| parse_json(@converter.convert(line), &block) end + rescue java.lang.IllegalStateException => e + if e.message == "input buffer full" && @decode_size_limit_bytes != "none" + yield event_factory.new_event("message" => data[0..@decode_size_limit_bytes.to_i - 1], "tags" => ["_jsonparsetoobigfailure"]) #TODO check the failure tag + else + # re-raise the error if doesn't correspond to the buffer overflow condition + raise e + end end def encode(event) diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index 581eb5b..0aa7a60 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -128,6 +128,13 @@ }.not_to raise_error end + it "should raise an error if the max bytes are exceeded" do + subject.decode(maximum_payload << "z") do |event| + expect(event.get("tags")).to include("_jsonparsetoobigfailure") + expect(event.get("message").size).to eq(decode_size_limit_bytes) + end + end + it "should raise an error if the max bytes are exceeded" do subject.decode(maximum_payload << "z") do |event| expect(event.get("tags")).to include("_jsonparse_too_big_failure") From 0db5a0ff24c0b947bfc5e60c5f8ccfc63f23a69b Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 3 Sep 2024 14:38:34 +0200 Subject: [PATCH 3/5] Bumped the version --- CHANGELOG.md | 3 +++ logstash-codec-json_lines.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e69a7f..64863f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.2.3 + - Update behaviour of `decode_size_limit_bytes` to do not apply any limitation to the length of a line, eventually tagging the event if it's set. [#45](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/45) + ## 3.2.2 - Fix: updated the way to check if the `decode_size_limit_bytes` has been explicitly customised. [#47](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/47) diff --git a/logstash-codec-json_lines.gemspec b/logstash-codec-json_lines.gemspec index 00a5a38..a3d9a04 100644 --- a/logstash-codec-json_lines.gemspec +++ b/logstash-codec-json_lines.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-json_lines' - s.version = '3.2.2' + s.version = '3.2.3' s.licenses = ['Apache License (2.0)'] s.summary = "Reads and writes newline-delimited JSON" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 9888e674753833fd125225e6fe0e0ee00b0ec455 Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 1 Oct 2024 14:15:24 +0200 Subject: [PATCH 4/5] Fixed test to verify 'message' form --- lib/logstash/codecs/json_lines.rb | 3 ++- spec/codecs/json_lines_spec.rb | 9 +-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index b985f3e..4e7db85 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -61,6 +61,7 @@ def register deprecation_logger.deprecated "The default value for `decode_size_limit_bytes`, currently at 512Mb, will be lowered in a future version to prevent Out of Memory errors from abnormally large messages or missing delimiters. Please set a value that reflects the largest expected message size (e.g. 20971520 for 20Mb)" end @buffer = FileWatch::BufferedTokenizer.new(@delimiter, @decode_size_limit_bytes) + puts "DNADBG>> delimiter is [#{@delimiter}]" @converter = LogStash::Util::Charset.new(@charset) @converter.logger = @logger end @@ -71,7 +72,7 @@ def decode(data, &block) end rescue java.lang.IllegalStateException => e if e.message == "input buffer full" && @decode_size_limit_bytes != "none" - yield event_factory.new_event("message" => data[0..@decode_size_limit_bytes.to_i - 1], "tags" => ["_jsonparsetoobigfailure"]) #TODO check the failure tag + yield event_factory.new_event("message" => "Payload bigger than #{@decode_size_limit_bytes} bytes", "tags" => ["_jsonparsetoobigfailure"]) else # re-raise the error if doesn't correspond to the buffer overflow condition raise e diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index 0aa7a60..1d6e49b 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -131,17 +131,10 @@ it "should raise an error if the max bytes are exceeded" do subject.decode(maximum_payload << "z") do |event| expect(event.get("tags")).to include("_jsonparsetoobigfailure") - expect(event.get("message").size).to eq(decode_size_limit_bytes) + expect(event.get("message")).to eq("Payload bigger than #{subject.decode_size_limit_bytes} bytes") end end - it "should raise an error if the max bytes are exceeded" do - subject.decode(maximum_payload << "z") do |event| - expect(event.get("tags")).to include("_jsonparse_too_big_failure") - expect(event.get("message").size).to eq(decode_size_limit_bytes) - end - end - it "should raise an error if the max bytes are exceeded" do expect { subject.decode(maximum_payload << "z") From 8c5234a0ba13959f94248b61e649e176acb642f0 Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 1 Oct 2024 14:29:15 +0200 Subject: [PATCH 5/5] Removed a test that verified the throw of an exception on buffer full condition, now instead the exception is raised producing a properly tagged event --- lib/logstash/codecs/json_lines.rb | 1 - spec/codecs/json_lines_spec.rb | 6 ------ 2 files changed, 7 deletions(-) diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index 4e7db85..e518a95 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -61,7 +61,6 @@ def register deprecation_logger.deprecated "The default value for `decode_size_limit_bytes`, currently at 512Mb, will be lowered in a future version to prevent Out of Memory errors from abnormally large messages or missing delimiters. Please set a value that reflects the largest expected message size (e.g. 20971520 for 20Mb)" end @buffer = FileWatch::BufferedTokenizer.new(@delimiter, @decode_size_limit_bytes) - puts "DNADBG>> delimiter is [#{@delimiter}]" @converter = LogStash::Util::Charset.new(@charset) @converter.logger = @logger end diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index 1d6e49b..7e7a96f 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -134,12 +134,6 @@ expect(event.get("message")).to eq("Payload bigger than #{subject.decode_size_limit_bytes} bytes") end end - - it "should raise an error if the max bytes are exceeded" do - expect { - subject.decode(maximum_payload << "z") - }.to raise_error(java.lang.IllegalStateException, "input buffer full") - end end end