diff --git a/Gemfile b/Gemfile index ccb9012..c48c3d1 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,3 @@ source 'https://rubygems.org' -gem 'rake' -gem 'gem_publisher' -gem 'archive-tar-minitar' +gemspec +gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5" diff --git a/Rakefile b/Rakefile index da6b220..4f4b858 100644 --- a/Rakefile +++ b/Rakefile @@ -4,3 +4,4 @@ task :default do system("rake -T") end +require "logstash/devutils/rake" diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 87c1c0c..0fc395c 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,13 +1,13 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '0.1.1' + s.version = '0.1.2' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." - s.description = "Stream events from files." + s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elasticsearch"] s.email = 'richard.pijnenburg@elasticsearch.com' - s.homepage = "http://logstash.net/" + s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html" s.require_paths = ["lib"] # Files @@ -17,7 +17,7 @@ Gem::Specification.new do |s| s.test_files = s.files.grep(%r{^(test|spec|features)/}) # Special flag to let us know this is actually a logstash plugin - s.metadata = { "logstash_plugin" => "true", "group" => "input" } + s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } # Gem dependencies s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0' @@ -26,5 +26,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'addressable' s.add_runtime_dependency 'filewatch', ['0.5.1'] + s.add_development_dependency 'logstash-devutils' end diff --git a/rakelib/publish.rake b/rakelib/publish.rake deleted file mode 100644 index 0ef58c0..0000000 --- a/rakelib/publish.rake +++ /dev/null @@ -1,9 +0,0 @@ -require "gem_publisher" - -desc "Publish gem to RubyGems.org" -task :publish_gem do |t| - gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first - gem = GemPublisher.publish_if_updated(gem_file, :rubygems) - puts "Published #{gem}" if gem -end - diff --git a/rakelib/vendor.rake b/rakelib/vendor.rake deleted file mode 100644 index 2135119..0000000 --- a/rakelib/vendor.rake +++ /dev/null @@ -1,169 +0,0 @@ -require "net/http" -require "uri" -require "digest/sha1" - -def vendor(*args) - return File.join("vendor", *args) -end - -directory "vendor/" => ["vendor"] do |task, args| - mkdir task.name -end - -def fetch(url, sha1, output) - - puts "Downloading #{url}" - actual_sha1 = download(url, output) - - if actual_sha1 != sha1 - fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')" - end -end # def fetch - -def file_fetch(url, sha1) - filename = File.basename( URI(url).path ) - output = "vendor/#{filename}" - task output => [ "vendor/" ] do - begin - actual_sha1 = file_sha1(output) - if actual_sha1 != sha1 - fetch(url, sha1, output) - end - rescue Errno::ENOENT - fetch(url, sha1, output) - end - end.invoke - - return output -end - -def file_sha1(path) - digest = Digest::SHA1.new - fd = File.new(path, "r") - while true - begin - digest << fd.sysread(16384) - rescue EOFError - break - end - end - return digest.hexdigest -ensure - fd.close if fd -end - -def download(url, output) - uri = URI(url) - digest = Digest::SHA1.new - tmp = "#{output}.tmp" - Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http| - request = Net::HTTP::Get.new(uri.path) - http.request(request) do |response| - fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code) - size = (response["content-length"].to_i || -1).to_f - count = 0 - File.open(tmp, "w") do |fd| - response.read_body do |chunk| - fd.write(chunk) - digest << chunk - if size > 0 && $stdout.tty? - count += chunk.bytesize - $stdout.write(sprintf("\r%0.2f%%", count/size * 100)) - end - end - end - $stdout.write("\r \r") if $stdout.tty? - end - end - - File.rename(tmp, output) - - return digest.hexdigest -rescue SocketError => e - puts "Failure while downloading #{url}: #{e}" - raise -ensure - File.unlink(tmp) if File.exist?(tmp) -end # def download - -def untar(tarball, &block) - require "archive/tar/minitar" - tgz = Zlib::GzipReader.new(File.open(tarball)) - # Pull out typesdb - tar = Archive::Tar::Minitar::Input.open(tgz) - tar.each do |entry| - path = block.call(entry) - next if path.nil? - parent = File.dirname(path) - - mkdir_p parent unless File.directory?(parent) - - # Skip this file if the output file is the same size - if entry.directory? - mkdir path unless File.directory?(path) - else - entry_mode = entry.instance_eval { @mode } & 0777 - if File.exists?(path) - stat = File.stat(path) - # TODO(sissel): Submit a patch to archive-tar-minitar upstream to - # expose headers in the entry. - entry_size = entry.instance_eval { @size } - # If file sizes are same, skip writing. - next if stat.size == entry_size && (stat.mode & 0777) == entry_mode - end - puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}" - File.open(path, "w") do |fd| - # eof? check lets us skip empty files. Necessary because the API provided by - # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an - # IO object. Something about empty files in this EntryStream causes - # IO.copy_stream to throw "can't convert nil into String" on JRuby - # TODO(sissel): File a bug about this. - while !entry.eof? - chunk = entry.read(16384) - fd.write(chunk) - end - #IO.copy_stream(entry, fd) - end - File.chmod(entry_mode, path) - end - end - tar.close - File.unlink(tarball) if File.file?(tarball) -end # def untar - -def ungz(file) - - outpath = file.gsub('.gz', '') - tgz = Zlib::GzipReader.new(File.open(file)) - begin - File.open(outpath, "w") do |out| - IO::copy_stream(tgz, out) - end - File.unlink(file) - rescue - File.unlink(outpath) if File.file?(outpath) - raise - end - tgz.close -end - -desc "Process any vendor files required for this plugin" -task "vendor" do |task, args| - - @files.each do |file| - download = file_fetch(file['url'], file['sha1']) - if download =~ /.tar.gz/ - prefix = download.gsub('.tar.gz', '').gsub('vendor/', '') - untar(download) do |entry| - if !file['files'].nil? - next unless file['files'].include?(entry.full_name.gsub(prefix, '')) - out = entry.full_name.split("/").last - end - File.join('vendor', out) - end - elsif download =~ /.gz/ - ungz(download) - end - end - -end diff --git a/spec/inputs/file_spec.rb b/spec/inputs/file_spec.rb index 462ade1..4414fad 100644 --- a/spec/inputs/file_spec.rb +++ b/spec/inputs/file_spec.rb @@ -1,6 +1,6 @@ # encoding: utf-8 -require "spec_helper" +require "logstash/devutils/rspec/spec_helper" require "tempfile" describe "inputs/file" do