From b111327d2a6c97b42e062dac83ac1ee5630244d4 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Tue, 15 Mar 2016 20:36:15 +0000 Subject: [PATCH] Fixes for #101 and #104 make explicit exit flushing, not to rely on close() side effects. fix failing specs NOTE auto_flush takes an argument More detail in the changelog change log change and remove rescue nil closes #101 closes #104 Fixes #112 --- CHANGELOG.md | 7 +++++++ lib/logstash/inputs/file.rb | 35 +++++++++++++++++++++++++++++------ logstash-input-file.gemspec | 4 ++-- spec/inputs/file_spec.rb | 8 ++------ spec/spec_helper.rb | 5 ++++- 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22b6aef..a5bd0a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.2.2 + - Fix for: Filewatch library complains if HOME or SINCEDB_PATH variables are unset. + - [Issue #101](https://github.com/logstash-plugins/logstash-input-file/issues/101) + - [PR, filewatch 78](https://github.com/jordansissel/ruby-filewatch/pull/78) introduces the fix + - [Issue, filewatch 76](https://github.com/jordansissel/ruby-filewatch/issues/76) + - Improve documentation on ignore_older and close_older options [#104](https://github.com/logstash-plugins/logstash-input-file/issues/104) Documentation + ## 2.2.1 - Fix spec failures on CI Linux builds (not seen on local OSX and Linux) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index f00eb44..7dab508 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -77,7 +77,7 @@ def accept(listener) end end if !method_defined?(:auto_flush) - def auto_flush + def auto_flush(*) end end end @@ -139,14 +139,14 @@ class LogStash::Inputs::File < LogStash::Inputs::Base # set the new line delimiter, defaults to "\n" config :delimiter, :validate => :string, :default => "\n" - # If this option is specified, when the file input discovers a file that - # was last modified before the specified timespan in seconds, the file is - # ignored. After it's discovery, if an ignored file is modified it is no + # When the file input discovers a file that was last modified + # before the specified timespan in seconds, the file is ignored. + # After it's discovery, if an ignored file is modified it is no # longer ignored and any new data is read. The default is 24 hours. config :ignore_older, :validate => :number, :default => 24 * 60 * 60 - # If this option is specified, the file input closes any files that were last - # read the specified timespan in seconds ago. + # The file input closes any files that were last read the specified + # timespan in seconds ago. # This has different implications depending on if a file is being tailed or # read. If tailing, and there is a large time gap in incoming data the file # can be closed (allowing other files to be opened) but will be queued for @@ -269,6 +269,10 @@ def dup_adding_state(line) end end + class FlushableListener < ListenerTail + attr_writer :path + end + def listener_for(path) # path is the identity ListenerTail.new(path, self) @@ -288,6 +292,7 @@ def run(queue) begin_tailing @queue = queue @tail.subscribe(self) + exit_flush end # def run def post_process_this(event) @@ -310,4 +315,22 @@ def stop @tail.quit end end + + private + + def exit_flush + listener = FlushableListener.new("none", self) + if @codec.identity_count.zero? + # using the base codec without identity/path info + @codec.base_codec.flush do |event| + begin + listener.process_event(event) + rescue => e + @logger.error("File Input: flush on exit downstream error", :exception => e) + end + end + else + @codec.flush_mapped(listener) + end + end end # class LogStash::Inputs::File diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 2646a07..43a4104 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '2.2.1' + s.version = '2.2.2' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -24,7 +24,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'addressable' - s.add_runtime_dependency 'filewatch', ['>= 0.8.0', '~> 0.8'] + s.add_runtime_dependency 'filewatch', ['>= 0.8.1', '~> 0.8'] s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.7'] s.add_development_dependency 'stud', ['~> 0.0.19'] diff --git a/spec/inputs/file_spec.rb b/spec/inputs/file_spec.rb index 19a5f8c..19632a8 100644 --- a/spec/inputs/file_spec.rb +++ b/spec/inputs/file_spec.rb @@ -181,15 +181,11 @@ expect(events.size).to eq(0) File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") } end - .then_after(0.1, "only one event is created, the last line is buffered") do - expect(events.size).to eq(1) - end - .then_after(0.1, "quit") do + .then_after(0.25, "quit") do subject.stop end + subject.run(events) - # stop flushes the second event - expect(events.size).to eq(2) event1 = events[0] expect(event1).not_to be_nil diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3aa87c3..850c275 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -47,9 +47,12 @@ def decode_accept(ctx, data, listener) def accept(listener) @tracer.push [:accept, true] end - def auto_flush() + def auto_flush(*) @tracer.push [:auto_flush, true] end + def flush(*) + @tracer.push [:flush, true] + end def close @tracer.push [:close, true] end