diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..3f4c785 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,57 @@ +name: CI + +on: + push: + branches: + - main + + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + name: Ruby ${{ matrix.ruby }} + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + matrix: + ruby: + - '2.7' + - '3.0' + - '3.1' + - '3.2' + + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Install dependencies + run: make init + - name: Run tests + run: make test + - name: Run lints + run: make lint + results: + if: ${{ always() }} + runs-on: ubuntu-latest + name: Final Results + needs: [test] + steps: + - run: exit 1 + # see https://stackoverflow.com/a/67532120/4907315 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8924f5f --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ +Gemfile.lock + +# rspec failure tracking +.rspec_status + +# Appraisals +/gemfiles diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..34c5164 --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..ff3c4a9 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,24 @@ +require: + - rubocop-rake + +AllCops: + TargetRubyVersion: 2.7 + NewCops: enable + +Style/StringLiterals: + Enabled: true + EnforcedStyle: double_quotes + +Style/StringLiteralsInInterpolation: + Enabled: true + EnforcedStyle: double_quotes + +Layout/LineLength: + Max: 120 + +Naming/FileName: + Exclude: ['lib/sidekiq-rescue.rb'] + +Style/FrozenStringLiteralComment: + Exclude: + - 'gemfiles/**/*' diff --git a/Appraisals b/Appraisals new file mode 100644 index 0000000..c62504a --- /dev/null +++ b/Appraisals @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +appraise "sidekiq-6.5.x" do + group :test do + gem "sidekiq", "~> 6.5.0" + end +end + +appraise "sidekiq-7.0.x" do + group :test do + gem "sidekiq", "~> 7.0.0" + end +end + +appraise "sidekiq-7.1.x" do + group :test do + gem "sidekiq", "~> 7.1.0" + end +end + +appraise "sidekiq-7.2.x" do + group :test do + gem "sidekiq", "~> 7.2.0" + end +end diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..143a1b0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## [Unreleased] + +## [0.1.0] - 2023-12-27 + +- Initial release diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..9307060 --- /dev/null +++ b/Gemfile @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in sidekiq_rescue.gemspec +gemspec + +gem "appraisal" +gem "rake", "~> 13.0" + +gem "rspec", "~> 3.0" + +gem "rubocop", "~> 1.21" +gem "rubocop-rake" +gem "rubocop-rspec" diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..a01c7ad --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Dmitrii Ivliev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..495ff57 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +PHONY: test lint + +init: + bundle install + bundle exec appraisal generate + bundle exec appraisal install + +test: + bundle exec appraisal rake spec + +lint: + bundle exec rubocop diff --git a/README.md b/README.md new file mode 100644 index 0000000..da444aa --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +# Sidekiq::Rescue + +[![Build Status](https://github.com/moofkit/sidekiq-rescue/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/moofkit/sidekiq-rescue/actions/workflows/main.yml) + +[Sidekiq](https://github.com/sidekiq/sidekiq) plugin to rescue jobs from expected errors and retry them later. + +## Installation + +Add this line to your application's Gemfile: + +```ruby +gem "sidekiq-rescue" +``` + +And then execute: + + $ bundle install + +Or install it yourself as: + + $ gem install sidekiq-rescue + +## Usage + +1. Add the middleware to your Sidekiq configuration: + +```ruby +Sidekiq.configure_server do |config| + config.server_middleware do |chain| + chain.add Sidekiq::Rescue::Middleware + end +end +``` + +2. Add DSL to your job: + +```ruby +class MyJob + include Sidekiq::Job + include Sidekiq::Rescue::DSL + + sidekiq_rescue ExpectedError + + def perform(*) + # ... + end +end +``` + +## Configuration + +You can configure the number of retries and the delay (in seconds) between retries: + +```ruby +class MyJob + include Sidekiq::Job + include Sidekiq::Rescue::DSL + + sidekiq_rescue ExpectedError, delay: 60, limit: 5 + + def perform(*) + # ... + end +end +``` + +The default values are: +- `delay`: 60 seconds +- `limit`: 5 retries + +Delay does not increase between retries (unlike Sidekiq standard retry mechanism). + +## Use cases + +Sidekiq::Rescue is useful when you want to retry jobs that failed due to expected errors and not spam your exception tracker with these errors. For example, you may want to retry a job that failed due to a network error or a temporary outage of a third party service, rather than a bug in your code. + +## Motivation + +Sidekiq provides a retry mechanism for jobs that failed due to unexpected errors. However, it does not provide a way to retry jobs that failed due to expected errors. This gem aims to fill this gap. +In addition, it provides a way to configure the number of retries and the delay between retries independently from the Sidekiq standard retry mechanism. + +## Supported Ruby versions + +This gem supports Ruby 2.7+ + +If something doesn't work on one of these versions, it's a bug + +## Supported Sidekiq versions + +This gem supports Sidekiq 6.5+. It may work with older versions, but it's not tested. + +If you need support for older versions, please open an issue + +## Development + +To install dependencies and run tests: +```bash +make init +make test +make lint +``` + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/moofkit/sidekiq_rescue. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..cca7175 --- /dev/null +++ b/Rakefile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rspec/core/rake_task" + +RSpec::Core::RakeTask.new(:spec) + +require "rubocop/rake_task" + +RuboCop::RakeTask.new + +task default: %i[spec rubocop] diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..27291fc --- /dev/null +++ b/bin/console @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require "sidekiq_rescue" + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +require "irb" +IRB.start(__FILE__) diff --git a/bin/setup b/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/lib/sidekiq-rescue.rb b/lib/sidekiq-rescue.rb new file mode 100644 index 0000000..064dc21 --- /dev/null +++ b/lib/sidekiq-rescue.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require "sidekiq_rescue" diff --git a/lib/sidekiq/rescue.rb b/lib/sidekiq/rescue.rb new file mode 100644 index 0000000..9ab81f7 --- /dev/null +++ b/lib/sidekiq/rescue.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Sidekiq + # Sidekiq::Rescue is a Sidekiq plugin which allows you to easily handle + # exceptions thrown by your jobs. + # + # To use Sidekiq::Rescue, you need to include Sidekiq::Rescue::DSL module + # in your job class and use the sidekiq_rescue class method to define + # exception handlers. + # + # class MyJob + # include Sidekiq::Job + # include Sidekiq::Rescue::DSL + # + # sidekiq_rescue NetworkError, delay: 60, limit: 10 + # + # def perform + # # ... + # end + # end + # + # Also it needs to be registered in Sidekiq server middleware chain: + # Sidekiq.configure_server do |config| + # config.server_middleware do |chain| + # chain.add Sidekiq::Rescue::ServerMiddleware + # ... + module Rescue + DEFAULT_DELAY = 60 + DEFAULT_LIMIT = 10 + + class << self + attr_writer :logger + + # Returns the logger instance. If no logger is set, it defaults to Sidekiq.logger. + # + # @return [Logger] The logger instance. + def logger + @logger ||= Sidekiq.logger + end + end + end +end diff --git a/lib/sidekiq/rescue/dsl.rb b/lib/sidekiq/rescue/dsl.rb new file mode 100644 index 0000000..41260ce --- /dev/null +++ b/lib/sidekiq/rescue/dsl.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Sidekiq + module Rescue + # This module is included into the job class to provide the DSL for + # configuring rescue options. + module DSL + def self.included(base) + base.extend(ClassMethods) + base.sidekiq_class_attribute(:sidekiq_rescue_options) + end + + # Module containing the DSL methods + module ClassMethods + # Configure rescue options for the job. + # @param error [StandardError] The error class to rescue. + # @param delay [Integer] The delay in seconds before retrying the job. + # @param limit [Integer] The maximum number of retries. + # @return [void] + # @raise [ArgumentError] if error is not a StandardError + # @example + # sidekiq_rescue NetworkError, delay: 60, limit: 10 + def sidekiq_rescue(error, **options) + raise ArgumentError, "error must be an ancestor of StandardError" unless error < StandardError + raise ArgumentError, "delay must be integer" if options[:delay] && !options[:delay].is_a?(Integer) + raise ArgumentError, "limit must be integer" if options[:limit] && !options[:limit].is_a?(Integer) + + self.sidekiq_rescue_options = { + error: error, + delay: Sidekiq::Rescue::DEFAULT_DELAY, + limit: Sidekiq::Rescue::DEFAULT_LIMIT + }.merge(options) + end + end + end + end +end diff --git a/lib/sidekiq/rescue/server_middleware.rb b/lib/sidekiq/rescue/server_middleware.rb new file mode 100644 index 0000000..d5bd035 --- /dev/null +++ b/lib/sidekiq/rescue/server_middleware.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Sidekiq + module Rescue + # Server middleware for sidekiq-rescue + # It is responsible for catching the errors and rescheduling the job + # according to the options provided + # @api private + class ServerMiddleware + include Sidekiq::ServerMiddleware + + def call(job_instance, job_payload, _queue, &block) + job_class = job_instance.class + options = job_class.sidekiq_rescue_options if job_class.respond_to?(:sidekiq_rescue_options) + if options + sidekiq_rescue(job_payload, **options, &block) + else + yield + end + end + + private + + def sidekiq_rescue(job_payload, delay:, limit:, error:, **) + yield + rescue *error => e + rescue_counter = job_payload["sidekiq_rescue_counter"].to_i + rescue_counter += 1 + raise e if rescue_counter > limit + + # NOTE: we use the retry counter to increase the jitter + # so that the jobs don't retry at the same time + # inspired by sidekiq https://github.com/sidekiq/sidekiq/blob/73c150d0430a8394cadb5cd49218895b113613a0/lib/sidekiq/job_retry.rb#L188 + jitter = rand(10) * rescue_counter + reschedule_at = Time.now.to_f + delay + jitter + + Sidekiq::Rescue.logger.info("[sidekiq_rescue] Job failed #{rescue_counter} times with error:" \ + "#{e.message}; rescheduling at #{reschedule_at}") + Sidekiq::Client.push(job_payload.merge("at" => reschedule_at, "sidekiq_rescue_counter" => rescue_counter)) + end + end + end +end diff --git a/lib/sidekiq/rescue/version.rb b/lib/sidekiq/rescue/version.rb new file mode 100644 index 0000000..6ebcec4 --- /dev/null +++ b/lib/sidekiq/rescue/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Sidekiq + module Rescue + VERSION = "0.1.0" + end +end diff --git a/lib/sidekiq_rescue.rb b/lib/sidekiq_rescue.rb new file mode 100644 index 0000000..e24600f --- /dev/null +++ b/lib/sidekiq_rescue.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require "sidekiq" +require_relative "sidekiq/rescue" +require_relative "sidekiq/rescue/version" +require_relative "sidekiq/rescue/dsl" +require_relative "sidekiq/rescue/server_middleware" diff --git a/sidekiq_rescue.gemspec b/sidekiq_rescue.gemspec new file mode 100644 index 0000000..1f96def --- /dev/null +++ b/sidekiq_rescue.gemspec @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require_relative "lib/sidekiq/rescue/version" + +Gem::Specification.new do |spec| + spec.name = "sidekiq_rescue" + spec.version = Sidekiq::Rescue::VERSION + spec.authors = ["Dmitrii Ivliev"] + spec.email = ["mail@ivda.dev"] + spec.license = "MIT" + + spec.summary = "Rescue Sidekiq jobs on expected error and reschedule them" + spec.homepage = "https://github.com/moofkit/sidekiq-rescue" + spec.required_ruby_version = ">= 2.7.0" + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md" + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + spec.files = Dir["{lib}/**/*"] + %w[LICENSE.txt Rakefile README.md CHANGELOG.md] + spec.require_paths = ["lib"] + + spec.add_dependency "sidekiq", ">= 6.5" + + spec.metadata["rubygems_mfa_required"] = "true" +end diff --git a/spec/.rubocop.yml b/spec/.rubocop.yml new file mode 100644 index 0000000..5160dd4 --- /dev/null +++ b/spec/.rubocop.yml @@ -0,0 +1,8 @@ +inherit_from: + - ../.rubocop.yml + +require: + - rubocop-rspec + +RSpec/MultipleExpectations: + Max: 2 diff --git a/spec/sidekiq/rescue/server_middleware_spec.rb b/spec/sidekiq/rescue/server_middleware_spec.rb new file mode 100644 index 0000000..ca12524 --- /dev/null +++ b/spec/sidekiq/rescue/server_middleware_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +RSpec.describe Sidekiq::Rescue::ServerMiddleware do + let(:middleware) { described_class.new } + let(:job_instance) { TestJob.new } + let(:job_payload) do + { + "class" => "TestJob", + "args" => [1], + "retry" => true, + "queue" => "default", + "jid" => "123", + "created_at" => Time.now.to_f, + "sidekiq_rescue_counter" => 0 + } + end + + context "with expected error" do + subject(:call_with_expected_error) { middleware.call(job_instance, job_payload, "default") { raise TestError } } + + it "reschedules the job on expected error and increments counter" do + allow(Sidekiq::Client).to receive(:push) + + call_with_expected_error + expect(Sidekiq::Client).to have_received(:push).with( + job_payload.merge("at" => anything, "sidekiq_rescue_counter" => 1) + ) + end + + it "suppress the error" do + expect { call_with_expected_error }.not_to raise_error + end + end + + context "with unexpected error" do + subject(:call_with_unexpected_error) do + middleware.call(job_instance, job_payload, "default") do + raise StandardError + end + end + + it "does not reschedule the job" do + allow(Sidekiq::Client).to receive(:push) + + expect { call_with_unexpected_error }.to raise_error(StandardError) + expect(Sidekiq::Client).not_to have_received(:push) + end + end +end diff --git a/spec/sidekiq/rescue_spec.rb b/spec/sidekiq/rescue_spec.rb new file mode 100644 index 0000000..75ac319 --- /dev/null +++ b/spec/sidekiq/rescue_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +RSpec.describe Sidekiq::Rescue do + it "has a version number" do + expect(Sidekiq::Rescue::VERSION).not_to be_nil + end + + it "provides a default sidekiq logger" do + expect(described_class.logger).to be_a(Sidekiq::Logger) + end + + it "allows to set a custom logger" do + logger = Logger.new($stdout) + described_class.logger = logger + expect(described_class.logger).to eq(logger) + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..7931030 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require "sidekiq_rescue" +Dir[File.join(__dir__, "support/**/*.rb")].sort.each { |f| require f } + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/spec/support/errors/test_error.rb b/spec/support/errors/test_error.rb new file mode 100644 index 0000000..22d6d86 --- /dev/null +++ b/spec/support/errors/test_error.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +TestError = Class.new(StandardError) diff --git a/spec/support/jobs/test_job.rb b/spec/support/jobs/test_job.rb new file mode 100644 index 0000000..2e7664b --- /dev/null +++ b/spec/support/jobs/test_job.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class TestJob + include Sidekiq::Job + include Sidekiq::Rescue::DSL + + sidekiq_rescue TestError + + def perform(*) + raise TestError + end +end