diff --git a/CHANGELOG.md b/CHANGELOG.md index d41b8bf..f3c6f07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ - Simplify workers memory calculation in PumaMemory‘s `get_total` method #81 - Add rubocop in gemspec and CI, with offenses corrected and unnecessary cops disabled. +- Add `pre_term`-like `rolling_pre_term` config for terminations caused by rolling restart (#86) +- Fix compatibility with ruby version 2.3.X (#87) ## 0.1.1 diff --git a/README.md b/README.md index cd39c71..3c44312 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ PumaWorkerKiller.config do |config| # PumaWorkerKiller: Consuming 54.34765625 mb with master and 2 workers. config.pre_term = -> (worker) { puts "Worker #{worker.inspect} being killed" } + config.rolling_pre_term = -> (worker) { puts "Worker #{worker.inspect} being killed by rolling restart" } end PumaWorkerKiller.start ``` @@ -139,6 +140,17 @@ PumaWorkerKiller: Rolling Restart. 5 workers consuming total: 650mb mb. Sending However you may want to collect more data, such as sending an event to an error collection service like rollbar or airbrake. The `pre_term` lambda gets called before any worker is killed by PWK for any reason. +### rolling_pre_term + +`config.rolling_pre_term` will be called just prior to worker termination by rolling restart when rolling restart is enabled. + +It is similar to `config.pre_term`. + +Difference: + +- `config.pre_term` is triggered only by terminations related with exceeding RAM +- `config.rolling_pre_term` is triggered only by terminations caused by enabled rolling restart + ### on_calculation `config.on_calculation` will be called every time Puma Worker Killer calculates memory usage (`config.frequency`). This may be useful for monitoring your total puma application memory usage, which can be contrasted with other application monitoring solutions. diff --git a/lib/puma_worker_killer.rb b/lib/puma_worker_killer.rb index 004d8d6..8710fa2 100644 --- a/lib/puma_worker_killer.rb +++ b/lib/puma_worker_killer.rb @@ -5,13 +5,16 @@ module PumaWorkerKiller extend self - attr_accessor :ram, :frequency, :percent_usage, :rolling_restart_frequency, :reaper_status_logs, :pre_term, :on_calculation + attr_accessor :ram, :frequency, :percent_usage, :rolling_restart_frequency, + :reaper_status_logs, :pre_term, :rolling_pre_term, :on_calculation + self.ram = 512 # mb self.frequency = 10 # seconds self.percent_usage = 0.99 # percent of RAM to use self.rolling_restart_frequency = 6 * 3600 # 6 hours in seconds self.reaper_status_logs = true self.pre_term = nil + self.rolling_pre_term = nil self.on_calculation = nil def config @@ -27,9 +30,9 @@ def start(frequency = self.frequency, reaper = self.reaper) enable_rolling_restart(rolling_restart_frequency) if rolling_restart_frequency end - def enable_rolling_restart(frequency = rolling_restart_frequency) + def enable_rolling_restart(frequency = self.rolling_restart_frequency) frequency += rand(0..10.0) # so all workers don't restart at the exact same time across multiple machines - AutoReap.new(frequency, RollingRestart.new).start + AutoReap.new(frequency, RollingRestart.new(nil, self.rolling_pre_term)).start end end diff --git a/lib/puma_worker_killer/puma_memory.rb b/lib/puma_worker_killer/puma_memory.rb index 3efbd04..a201d15 100644 --- a/lib/puma_worker_killer/puma_memory.rb +++ b/lib/puma_worker_killer/puma_memory.rb @@ -52,7 +52,7 @@ def largest_worker_memory # Will refresh @workers def get_total(workers = set_workers) master_memory = GetProcessMem.new(Process.pid).mb - worker_memory = workers.values.sum + worker_memory = workers.values.inject(:+) || 0 worker_memory + master_memory end alias get_total_memory get_total diff --git a/lib/puma_worker_killer/rolling_restart.rb b/lib/puma_worker_killer/rolling_restart.rb index 3018148..e4e64ee 100644 --- a/lib/puma_worker_killer/rolling_restart.rb +++ b/lib/puma_worker_killer/rolling_restart.rb @@ -2,8 +2,9 @@ module PumaWorkerKiller class RollingRestart - def initialize(master = nil) + def initialize(master = nil, rolling_pre_term = nil) @cluster = PumaWorkerKiller::PumaMemory.new(master) + @rolling_pre_term = rolling_pre_term end # used for tes @@ -18,6 +19,8 @@ def reap(seconds_between_worker_kill = 60) @cluster.workers.each do |worker, _ram| @cluster.master.log "PumaWorkerKiller: Rolling Restart. #{@cluster.workers.count} workers consuming total: #{total_memory} mb. Sending TERM to pid #{worker.pid}." + @rolling_pre_term.call(worker) unless @rolling_pre_term.nil? + worker.term sleep seconds_between_worker_kill end diff --git a/test/fixtures/rolling_pre_term.ru b/test/fixtures/rolling_pre_term.ru new file mode 100644 index 0000000..2ef9582 --- /dev/null +++ b/test/fixtures/rolling_pre_term.ru @@ -0,0 +1,8 @@ +load File.expand_path("../fixture_helper.rb", __FILE__) + +PumaWorkerKiller.config do |config| + config.rolling_pre_term = lambda { |worker| puts("About to terminate (rolling) worker: #{worker.pid}") } +end +PumaWorkerKiller.enable_rolling_restart(1) # 1 second + +run HelloWorldApp diff --git a/test/puma_worker_killer_test.rb b/test/puma_worker_killer_test.rb index 40362aa..5578683 100644 --- a/test/puma_worker_killer_test.rb +++ b/test/puma_worker_killer_test.rb @@ -88,4 +88,17 @@ def test_rolling_restart_worker_kill_check assert term_ids.sort == term_ids.uniq.sort end end + + def test_rolling_pre_term + file = fixture_path.join("rolling_pre_term.ru") + port = 0 + command = "bundle exec puma #{ file } -t 1:1 -w 2 --preload --debug -p #{ port }" + puts command.inspect + options = { wait_for: "booted", timeout: 15, env: { } } + + WaitForIt.new(command, options) do |spawn| + assert_contains(spawn, "Rolling Restart") + assert_contains(spawn, "About to terminate (rolling) worker:") # defined in rolling_pre_term.ru + end + end end