From 79ee874a075dd7e91fa6fc80f0cd165f37397aa0 Mon Sep 17 00:00:00 2001 From: Joshua Morris Date: Fri, 18 Dec 2015 22:45:09 +0000 Subject: [PATCH 1/6] Remove clearing custom fields after first use --- lib/logstasher/action_view/log_subscriber.rb | 1 - lib/logstasher/active_record/log_subscriber.rb | 1 - lib/logstasher/active_support/log_subscriber.rb | 1 - 3 files changed, 3 deletions(-) diff --git a/lib/logstasher/action_view/log_subscriber.rb b/lib/logstasher/action_view/log_subscriber.rb index dc8d215..4e73c7a 100644 --- a/lib/logstasher/action_view/log_subscriber.rb +++ b/lib/logstasher/action_view/log_subscriber.rb @@ -62,7 +62,6 @@ def runtimes(event) def extract_custom_fields(data) custom_fields = (!LogStasher.custom_fields.empty? && data.extract!(*LogStasher.custom_fields)) || {} - LogStasher.custom_fields.clear custom_fields end end diff --git a/lib/logstasher/active_record/log_subscriber.rb b/lib/logstasher/active_record/log_subscriber.rb index 3582dc4..6b6a7f2 100644 --- a/lib/logstasher/active_record/log_subscriber.rb +++ b/lib/logstasher/active_record/log_subscriber.rb @@ -52,7 +52,6 @@ def extract_sql(data) def extract_custom_fields(data) custom_fields = (!LogStasher.custom_fields.empty? && data.extract!(*LogStasher.custom_fields)) || {} - LogStasher.custom_fields.clear custom_fields end end diff --git a/lib/logstasher/active_support/log_subscriber.rb b/lib/logstasher/active_support/log_subscriber.rb index 8ba8134..3c94929 100644 --- a/lib/logstasher/active_support/log_subscriber.rb +++ b/lib/logstasher/active_support/log_subscriber.rb @@ -98,7 +98,6 @@ def extract_exception(payload) def extract_custom_fields(payload) custom_fields = (!LogStasher.custom_fields.empty? && payload.extract!(*LogStasher.custom_fields)) || {} - LogStasher.custom_fields.clear custom_fields end end From 6a7b771fa00c705ff8472c1014e995264b6ccb93 Mon Sep 17 00:00:00 2001 From: Joshua Morris Date: Thu, 7 Jan 2016 05:17:53 +0000 Subject: [PATCH 2/6] Added an optional YAML file to configure logstasher (config/logstasher.yml). Also allowed for disabling individual log subscribers e.g. disable the view log subscriber --- README.md | 27 ++++++++++++++++- lib/logstasher.rb | 12 ++++---- .../active_record/log_subscriber.rb | 1 - lib/logstasher/railtie.rb | 29 +++++++++++++++++++ .../logstasher.yml.sample | 7 +++++ 5 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 sample_logstash_configurations/logstasher.yml.sample diff --git a/README.md b/README.md index 184be70..33d8b14 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ In your Gemfile: # Enable the logstasher logs for the current environment config.logstasher.enabled = true + # Each of the following lines are optional. If you want to selectively disable log subscribers. + config.logstasher.controller_enabled = false + config.logstasher.mailer_enabled = false + config.logstasher.record_enabled = false + config.logstasher.view_enabled = false + # This line is optional if you do not want to suppress app logs in your .log config.logstasher.suppress_app_log = false @@ -66,10 +72,29 @@ In your Gemfile: # This line is optional if you do not want to log the backtrace of exceptions config.logstasher.backtrace = false - + # This line is optional, defaults to log/logstasher_.log config.logstasher.logger_path = 'log/logstasher.log' +## Optionally use config/logstasher.yml (overrides `.rb`) + +Has the same optional fields as the `.rb` + + development: + enabled: false + controller_enabled: true + mailer_enabled: false + record_enabled: true + view_enabled: true + suppress_app_log: false + production: + enabled: false + controller_enabled: true + mailer_enabled: true + record_enabled: false + view_enabled: false + suppress_app_log: false + ## Logging params hash Logstasher can be configured to log the contents of the params hash. When enabled, the contents of the params hash (minus the ActionController internal params) diff --git a/lib/logstasher.rb b/lib/logstasher.rb index 33e0385..866c943 100644 --- a/lib/logstasher.rb +++ b/lib/logstasher.rb @@ -14,7 +14,7 @@ module LogStasher STORE_KEY = :logstasher_data REQUEST_CONTEXT_KEY = :logstasher_request_context - attr_accessor :logger, :logger_path, :enabled, :log_controller_parameters, :source, :backtrace, + attr_accessor :logger, :logger_path, :enabled, :log_controller_parameters, :source, :backtrace, :controller_monkey_patch # Setting the default to 'unknown' to define the default behaviour @source = 'unknown' @@ -87,15 +87,15 @@ def clear_request_context def setup_before(config) require 'logstash-event' self.enabled = config.enabled - LogStasher::ActiveSupport::LogSubscriber.attach_to :action_controller - LogStasher::ActiveSupport::MailerLogSubscriber.attach_to :action_mailer - LogStasher::ActiveRecord::LogSubscriber.attach_to :active_record - LogStasher::ActionView::LogSubscriber.attach_to :action_view + LogStasher::ActiveSupport::LogSubscriber.attach_to :action_controller if config.controller_enabled + LogStasher::ActiveSupport::MailerLogSubscriber.attach_to :action_mailer if config.mailer_enabled + LogStasher::ActiveRecord::LogSubscriber.attach_to :active_record if config.record_enabled + LogStasher::ActionView::LogSubscriber.attach_to :action_view if config.view_enabled end def setup(config) # Path instrumentation class to insert our hook - if (! config.controller_monkey_patch && config.controller_monkey_patch != false) || config.controller_monkey_patch == true + if (! config.controller_monkey_patch && config.controller_monkey_patch != false) || config.controller_monkey_patch == true require 'logstasher/rails_ext/action_controller/metal/instrumentation' end self.suppress_app_logs(config) diff --git a/lib/logstasher/active_record/log_subscriber.rb b/lib/logstasher/active_record/log_subscriber.rb index 6b6a7f2..eaa9022 100644 --- a/lib/logstasher/active_record/log_subscriber.rb +++ b/lib/logstasher/active_record/log_subscriber.rb @@ -21,7 +21,6 @@ def logger def logstash_event(event) data = event.payload - return unless logger.debug? return if 'SCHEMA' == data[:name] data.merge! runtimes(event) diff --git a/lib/logstasher/railtie.rb b/lib/logstasher/railtie.rb index 8b20515..545b04e 100644 --- a/lib/logstasher/railtie.rb +++ b/lib/logstasher/railtie.rb @@ -1,13 +1,42 @@ require 'rails/railtie' require 'action_view/log_subscriber' require 'action_controller/log_subscriber' +require 'socket' module LogStasher class Railtie < Rails::Railtie config.logstasher = ::ActiveSupport::OrderedOptions.new config.logstasher.enabled = false + # Set up the default logging options + config.logstasher.controller_enabled = true + config.logstasher.mailer_enabled = true + config.logstasher.record_enabled = false + config.logstasher.view_enabled = true + + # Try loading the config/logstasher.yml if present + env = Rails.env || 'development' + config_file = File.expand_path "./config/logstasher.yml" + + # Load and ERB templating of YAML files + LOGSTASHER = File.exists?(config_file) ? YAML.load(ERB.new(File.read(config_file)).result)[env].symbolize_keys : nil + initializer :logstasher, :before => :load_config_initializers do |app| + if LOGSTASHER.present? + # Enable the logstasher logs for the current environment + app.config.logstasher.enabled = LOGSTASHER[:enabled] if LOGSTASHER.key? :enabled + app.config.logstasher.controller_enabled = LOGSTASHER[:controller_enabled] if LOGSTASHER.key? :controller_enabled + app.config.logstasher.mailer_enabled = LOGSTASHER[:mailer_enabled] if LOGSTASHER.key? :mailer_enabled + app.config.logstasher.record_enabled = LOGSTASHER[:record_enabled] if LOGSTASHER.key? :record_enabled + app.config.logstasher.view_enabled = LOGSTASHER[:view_enabled] if LOGSTASHER.key? :view_enabled + # + # # This line is optional if you do not want to suppress app logs in your .log + app.config.logstasher.suppress_app_log = LOGSTASHER[:suppress_app_log] if LOGSTASHER.key? :suppress_app_log + # + # # This line is optional, it allows you to set a custom value for the @source field of the log event + app.config.logstasher.source = LOGSTASHER[:source].present? ? LOGSTASHER[:source] : IPSocket.getaddress(Socket.gethostname) + end + app.config.action_dispatch.rack_cache[:verbose] = false if app.config.action_dispatch.rack_cache LogStasher.setup_before(app.config.logstasher) if app.config.logstasher.enabled end diff --git a/sample_logstash_configurations/logstasher.yml.sample b/sample_logstash_configurations/logstasher.yml.sample new file mode 100644 index 0000000..c068e31 --- /dev/null +++ b/sample_logstash_configurations/logstasher.yml.sample @@ -0,0 +1,7 @@ +development: + enabled: true + controller_enabled: true + mailer_enabled: false + record_enabled: false + view_enabled: false + suppress_app_log: false From 08c56012963fbc178d4bf23775bbd75d2b772903 Mon Sep 17 00:00:00 2001 From: Joshua Morris Date: Thu, 7 Jan 2016 05:19:40 +0000 Subject: [PATCH 3/6] Added an optional YAML file to configure logstasher (config/logstasher.yml). Also allowed for disabling individual log subscribers e.g. disable the view log subscriber --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33d8b14..ed19376 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,14 @@ In your Gemfile: Has the same optional fields as the `.rb` development: - enabled: false + enabled: true controller_enabled: true mailer_enabled: false record_enabled: true view_enabled: true suppress_app_log: false production: - enabled: false + enabled: true controller_enabled: true mailer_enabled: true record_enabled: false From 45c993155d9be679a86f240851d733b2b7ff91ad Mon Sep 17 00:00:00 2001 From: Joshua Morris Date: Sun, 10 Jan 2016 22:26:13 +0000 Subject: [PATCH 4/6] Added the ability to specify common configurations across all environments, that can then be overriden per environment if you want --- README.md | 14 ++++++-------- lib/logstasher/railtie.rb | 38 ++++++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ed19376..06d1821 100644 --- a/README.md +++ b/README.md @@ -78,22 +78,20 @@ In your Gemfile: ## Optionally use config/logstasher.yml (overrides `.rb`) -Has the same optional fields as the `.rb` +Has the same optional fields as the `.rb`. You can specify common configurations that are then overriden by environment specific configurations: + controller_enabled: true + mailer_enabled: false + record_enabled: false + view_enabled: true + suppress_app_log: false development: enabled: true - controller_enabled: true - mailer_enabled: false record_enabled: true - view_enabled: true - suppress_app_log: false production: enabled: true - controller_enabled: true mailer_enabled: true - record_enabled: false view_enabled: false - suppress_app_log: false ## Logging params hash diff --git a/lib/logstasher/railtie.rb b/lib/logstasher/railtie.rb index 545b04e..79e5363 100644 --- a/lib/logstasher/railtie.rb +++ b/lib/logstasher/railtie.rb @@ -15,26 +15,18 @@ class Railtie < Rails::Railtie config.logstasher.view_enabled = true # Try loading the config/logstasher.yml if present - env = Rails.env || 'development' + env = Rails.env.to_sym || :development config_file = File.expand_path "./config/logstasher.yml" # Load and ERB templating of YAML files - LOGSTASHER = File.exists?(config_file) ? YAML.load(ERB.new(File.read(config_file)).result)[env].symbolize_keys : nil + LOGSTASHER = File.exists?(config_file) ? YAML.load(ERB.new(File.read(config_file)).result).symbolize_keys : nil initializer :logstasher, :before => :load_config_initializers do |app| if LOGSTASHER.present? - # Enable the logstasher logs for the current environment - app.config.logstasher.enabled = LOGSTASHER[:enabled] if LOGSTASHER.key? :enabled - app.config.logstasher.controller_enabled = LOGSTASHER[:controller_enabled] if LOGSTASHER.key? :controller_enabled - app.config.logstasher.mailer_enabled = LOGSTASHER[:mailer_enabled] if LOGSTASHER.key? :mailer_enabled - app.config.logstasher.record_enabled = LOGSTASHER[:record_enabled] if LOGSTASHER.key? :record_enabled - app.config.logstasher.view_enabled = LOGSTASHER[:view_enabled] if LOGSTASHER.key? :view_enabled - # - # # This line is optional if you do not want to suppress app logs in your .log - app.config.logstasher.suppress_app_log = LOGSTASHER[:suppress_app_log] if LOGSTASHER.key? :suppress_app_log - # - # # This line is optional, it allows you to set a custom value for the @source field of the log event - app.config.logstasher.source = LOGSTASHER[:source].present? ? LOGSTASHER[:source] : IPSocket.getaddress(Socket.gethostname) + # process common configs + LogStasher.process_config(app.config.logstasher, LOGSTASHER) + # process environment specific configs + LogStasher.process_config(app.config.logstasher, LOGSTASHER[env].symbolize_keys) if LOGSTASHER.key? env end app.config.action_dispatch.rack_cache[:verbose] = false if app.config.action_dispatch.rack_cache @@ -47,4 +39,22 @@ class Railtie < Rails::Railtie end end end + + def process_config(config, yml_config) + # Enable the logstasher logs for the current environment + config.enabled = yml_config[:enabled] if yml_config.key? :enabled + config.controller_enabled = yml_config[:controller_enabled] if yml_config.key? :controller_enabled + config.mailer_enabled = yml_config[:mailer_enabled] if yml_config.key? :mailer_enabled + config.record_enabled = yml_config[:record_enabled] if yml_config.key? :record_enabled + config.view_enabled = yml_config[:view_enabled] if yml_config.key? :view_enabled + # + # # This line is optional if you do not want to suppress app logs in your .log + config.suppress_app_log = yml_config[:suppress_app_log] if yml_config.key? :suppress_app_log + # + # # This line is optional, it allows you to set a custom value for the @source field of the log event + config.source = yml_config[:source].present? ? yml_config[:source] : IPSocket.getaddress(Socket.gethostname) + + config.backtrace = yml_config[:backtrace] if yml_config.key? :backtrace + config.logger_path = yml_config[:logger_path] if yml_config.key? :logger_path + end end From 6f7cf2e31c28afef9234fa69616f0f404f7abce9 Mon Sep 17 00:00:00 2001 From: Joshua Morris Date: Fri, 15 Jan 2016 15:44:08 +0000 Subject: [PATCH 5/6] Update sample yaml with global options --- .../logstasher.yml.sample | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sample_logstash_configurations/logstasher.yml.sample b/sample_logstash_configurations/logstasher.yml.sample index c068e31..480ef2c 100644 --- a/sample_logstash_configurations/logstasher.yml.sample +++ b/sample_logstash_configurations/logstasher.yml.sample @@ -1,7 +1,13 @@ +controller_enabled: true +mailer_enabled: false +record_enabled: false +view_enabled: true +suppress_app_log: false development: enabled: true - controller_enabled: true - mailer_enabled: false - record_enabled: false + record_enabled: true +production: + enabled: true + mailer_enabled: true view_enabled: false - suppress_app_log: false + From 40b68fe33d98536924ab99f8ad22327de1ccbd7f Mon Sep 17 00:00:00 2001 From: Joshua Morris Date: Tue, 16 Feb 2016 00:32:20 +0000 Subject: [PATCH 6/6] Fixed tests for config _enabled options --- spec/lib/logstasher_spec.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/spec/lib/logstasher_spec.rb b/spec/lib/logstasher_spec.rb index 4d22c97..067b370 100644 --- a/spec/lib/logstasher_spec.rb +++ b/spec/lib/logstasher_spec.rb @@ -114,14 +114,15 @@ expect(LogStasher::ActionView::LogSubscriber).to receive(:attach_to).with(:action_view) expect(LogStasher).to receive(:require).with('logstash-event') end - + end shared_examples 'setup' do let(:logstasher_source) { nil } - let(:logstasher_config) { double(:enabled => true, + let(:logstasher_config) { double(:enabled => true, :logger => logger, :log_level => 'warn', :log_controller_parameters => nil, :source => logstasher_source, :logger_path => logger_path, :backtrace => true, - :controller_monkey_patch => true) } + :controller_monkey_patch => true, :controller_enabled => true, + :mailer_enabled => true, :record_enabled => false, :view_enabled => true) } let(:config) { double(:logstasher => logstasher_config) } let(:app) { double(:config => config) } before do @@ -301,7 +302,7 @@ end end end - + describe ".enabled?" do it "returns false if not enabled" do expect(LogStasher).to receive(:enabled).and_return(false) @@ -311,12 +312,12 @@ expect(LogStasher.enabled?).to be true end end - + describe ".called_as_rake?" do it "returns false if not called as rake" do expect(LogStasher.called_as_rake?).to be false end - + it "returns true if called as rake" do expect(File).to receive(:basename).with($0).and_return('rake') expect(LogStasher.called_as_rake?).to be true @@ -327,7 +328,7 @@ it "does not touch request_context if not called as rake" do expect(LogStasher.request_context).to be_empty end - + it "sets request_context accordingly if called as rake" do expect(LogStasher).to receive(:called_as_rake?).and_return(true) expect(Rake.application).to receive(:top_level_tasks).and_return(['mytask']) @@ -341,7 +342,7 @@ it "returns false if not called as console" do expect(LogStasher.called_as_console?).to be false end - + it "returns true if called as rake" do require 'rails/commands/console' expect(LogStasher.called_as_console?).to be true @@ -352,7 +353,7 @@ it "does not touch request_context if not called as console" do expect(LogStasher.request_context).to be_empty end - + it "sets request_context accordingly if called as console" do require 'rails/commands/console' expect(LogStasher).to receive(:called_as_console?).and_return(true)