From 5614680a6c95be401e050a68919749e0d2c81fd1 Mon Sep 17 00:00:00 2001 From: Andrejs Date: Sat, 17 Feb 2024 09:39:40 +0100 Subject: [PATCH] Add option to configure exception class for marking test as failed (#531) * Add option to configure exception class for marking test as failed * Document custom failure exception class --- allure-cucumber/README.md | 13 +++++++++++++ allure-cucumber/lib/allure_cucumber/formatter.rb | 3 ++- .../spec/unit/formatter_test_step_stopped_spec.rb | 4 ++-- allure-rspec/README.md | 12 ++++++++++++ allure-ruby-commons/lib/allure-ruby-commons.rb | 2 +- .../lib/allure_ruby_commons/config.rb | 10 +++++++++- .../lib/allure_ruby_commons/result_utils.rb | 2 +- 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/allure-cucumber/README.md b/allure-cucumber/README.md index aa03557b..c8bd7e32 100644 --- a/allure-cucumber/README.md +++ b/allure-cucumber/README.md @@ -48,6 +48,7 @@ AllureCucumber.configure do |config| config.logging_level = Logger::INFO config.logger = Logger.new($stdout, Logger::DEBUG) config.environment = "staging" + config.failure_exception = RSpec::Expectations::ExpectationNotMetError # these are used for creating links to bugs or test cases where {} is replaced with keys of relevant items config.link_tms_pattern = "http://www.jira.com/browse/{}" @@ -110,6 +111,18 @@ Example: Additional special tags exists for setting status detail of test scenarios, allure will pick up following tags: `@flaky`, `@known` and `@muted` +### Custom failure exception + +Allure report will mark steps and tests as either `Failed` or `Broken` based on exception class that was raised. By default, `RSpec::Expectations::ExpectationNotMetError` exception will mark test as `Failed` and all other exceptions will mark test as `Broken`. + +Custom failure exception class can be configured: + +```ruby +AllureCucumber.configure do |config| + config.failure_exception = MyCustomFailedException +end +``` + ## Usage Use `--format AllureCucumber::CucumberFormatter --out where/you-want-results` while running cucumber or add it to `cucumber.yml`. Note that cucumber `--out` option overrides `results_directory` set via `Allure.configure` method. diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index 8ec5e24e..b2333234 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -71,9 +71,10 @@ def on_test_step_started(event) # @param [Cucumber::Events::TestStepFinished] event # @return [void] def on_test_step_finished(event) + status = ALLURE_STATUS.fetch(event.result.to_sym, Allure::Status::BROKEN) update_block = proc do |step| step.stage = Allure::Stage::FINISHED - step.status = ALLURE_STATUS.fetch(event.result.to_sym, Allure::Status::BROKEN) + step.status = event.result.failed? ? Allure::ResultUtils.status(event.result&.exception) : status end event.test_step.hook? ? handle_hook_finished(event.test_step, update_block) : handle_step_finished(update_block) diff --git a/allure-cucumber/spec/unit/formatter_test_step_stopped_spec.rb b/allure-cucumber/spec/unit/formatter_test_step_stopped_spec.rb index 1a43e5bb..3519d859 100644 --- a/allure-cucumber/spec/unit/formatter_test_step_stopped_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_step_stopped_spec.rb @@ -37,7 +37,7 @@ expect(lifecycle).to have_received(:update_test_step).with(no_args).once do |&arg| arg.call(@step) end - expect(@step.status).to eq(Allure::Status::FAILED) + expect(@step.status).to eq(Allure::Status::BROKEN) end it "with skipped status is updated" do @@ -119,7 +119,7 @@ expect(lifecycle).to have_received(:update_fixture).with(no_args).once do |&arg| arg.call(@step) - expect(@step.status).to eq(Allure::Status::FAILED) + expect(@step.status).to eq(Allure::Status::BROKEN) end end end diff --git a/allure-rspec/README.md b/allure-rspec/README.md index d7879855..0b2147d2 100644 --- a/allure-rspec/README.md +++ b/allure-rspec/README.md @@ -207,6 +207,18 @@ it "some test case 2", story: "user story" do end ``` +### Custom failure exception + +Allure report will mark steps and tests as either `Failed` or `Broken` based on exception class that was raised. By default, `RSpec::Expectations::ExpectationNotMetError` exception will mark test as `Failed` and all other exceptions will mark test as `Broken`. + +Custom failure exception class can be configured: + +```ruby +AllureRspec.configure do |config| + config.failure_exception = MyCustomFailedException +end +``` + ### Custom actions Rspec example object has access to [Allure](https://www.rubydoc.info/github/allure-framework/allure-ruby/Allure) helper methods. diff --git a/allure-ruby-commons/lib/allure-ruby-commons.rb b/allure-ruby-commons/lib/allure-ruby-commons.rb index c34171d5..b7ca6b7d 100644 --- a/allure-ruby-commons/lib/allure-ruby-commons.rb +++ b/allure-ruby-commons/lib/allure-ruby-commons.rb @@ -225,7 +225,7 @@ def run_step(name) lifecycle.update_test_step { |step| step.status = Status::PASSED } result - rescue StandardError, RSpec::Expectations::ExpectationNotMetError => e + rescue StandardError, configuration.failure_exception => e lifecycle.update_test_step do |step| step.status = ResultUtils.status(e) step.status_details = ResultUtils.status_details(e) diff --git a/allure-ruby-commons/lib/allure_ruby_commons/config.rb b/allure-ruby-commons/lib/allure_ruby_commons/config.rb index c6fc1c9c..7681e6c4 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/config.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/config.rb @@ -2,6 +2,7 @@ require "logger" require "singleton" +require "rspec/expectations" module Allure # Allure configuration class @@ -11,7 +12,7 @@ class Config # @return [Array] valid log levels LOGLEVELS = %w[DEBUG INFO WARN ERROR FATAL UNKNOWN].freeze - attr_writer :environment, :logger + attr_writer :environment, :logger, :failure_exception attr_accessor :results_directory, :logging_level, @@ -41,5 +42,12 @@ def environment def logger @logger ||= Logger.new($stdout, level: logging_level) end + + # Exception class that corresponds to test failure + # + # @return [Class] + def failure_exception + @failure_exception ||= RSpec::Expectations::ExpectationNotMetError + end end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb index 586955ae..be9dc961 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb @@ -152,7 +152,7 @@ def issue_link(name, value, link_pattern) # @param [Exception] exception # @return [Symbol] def status(exception) - exception.is_a?(RSpec::Expectations::ExpectationNotMetError) ? Status::FAILED : Status::BROKEN + exception.is_a?(Allure.configuration.failure_exception) ? Status::FAILED : Status::BROKEN end # Get exception status detail