Add option to configure exception class for marking test as failed (#531

)

* Add option to configure exception class for marking test as failed

* Document custom failure exception class

allure-cucumber/README.md

@@ -48,6 +48,7 @@ AllureCucumber.configure do |config|
   config.logging_level = Logger::INFO
   config.logger = Logger.new($stdout, Logger::DEBUG)
   config.environment = "staging"
+  config.failure_exception = RSpec::Expectations::ExpectationNotMetError
 
   # these are used for creating links to bugs or test cases where {} is replaced with keys of relevant items
   config.link_tms_pattern = "http://www.jira.com/browse/{}"
@@ -110,6 +111,18 @@ Example:
 
 Additional special tags exists for setting status detail of test scenarios, allure will pick up following tags: `@flaky`, `@known` and `@muted`
 
+### Custom failure exception
+
+Allure report will mark steps and tests as either `Failed` or `Broken` based on exception class that was raised. By default, `RSpec::Expectations::ExpectationNotMetError` exception will mark test as `Failed` and all other exceptions will mark test as `Broken`.
+
+Custom failure exception class can be configured:
+
+```ruby
+AllureCucumber.configure do |config|
+  config.failure_exception = MyCustomFailedException
+end
+```
+
 ## Usage
 
 Use `--format AllureCucumber::CucumberFormatter --out where/you-want-results` while running cucumber or add it to `cucumber.yml`. Note that cucumber `--out` option overrides `results_directory` set via `Allure.configure` method.

allure-cucumber/lib/allure_cucumber/formatter.rb

@@ -71,9 +71,10 @@ def on_test_step_started(event)
     # @param [Cucumber::Events::TestStepFinished] event
     # @return [void]
     def on_test_step_finished(event)
+      status = ALLURE_STATUS.fetch(event.result.to_sym, Allure::Status::BROKEN)
       update_block = proc do |step|
         step.stage = Allure::Stage::FINISHED
-        step.status = ALLURE_STATUS.fetch(event.result.to_sym, Allure::Status::BROKEN)
+        step.status = event.result.failed? ? Allure::ResultUtils.status(event.result&.exception) : status
       end
 
       event.test_step.hook? ? handle_hook_finished(event.test_step, update_block) : handle_step_finished(update_block)

allure-cucumber/spec/unit/formatter_test_step_stopped_spec.rb

@@ -37,7 +37,7 @@
       expect(lifecycle).to have_received(:update_test_step).with(no_args).once do |&arg|
         arg.call(@step)
       end
-      expect(@step.status).to eq(Allure::Status::FAILED)
+      expect(@step.status).to eq(Allure::Status::BROKEN)
     end
 
     it "with skipped status is updated" do
@@ -119,7 +119,7 @@
 
       expect(lifecycle).to have_received(:update_fixture).with(no_args).once do |&arg|
         arg.call(@step)
-        expect(@step.status).to eq(Allure::Status::FAILED)
+        expect(@step.status).to eq(Allure::Status::BROKEN)
       end
     end
   end

allure-rspec/README.md

@@ -207,6 +207,18 @@ it "some test case 2", story: "user story" do
 end
 ```
 
+### Custom failure exception
+
+Allure report will mark steps and tests as either `Failed` or `Broken` based on exception class that was raised. By default, `RSpec::Expectations::ExpectationNotMetError` exception will mark test as `Failed` and all other exceptions will mark test as `Broken`.
+
+Custom failure exception class can be configured:
+
+```ruby
+AllureRspec.configure do |config|
+  config.failure_exception = MyCustomFailedException
+end
+```
+
 ### Custom actions
 
 Rspec example object has access to [Allure](https://www.rubydoc.info/github/allure-framework/allure-ruby/Allure) helper methods.

allure-ruby-commons/lib/allure-ruby-commons.rb

@@ -225,7 +225,7 @@ def run_step(name)
     lifecycle.update_test_step { |step| step.status = Status::PASSED }
 
     result
-  rescue StandardError, RSpec::Expectations::ExpectationNotMetError => e
+  rescue StandardError, configuration.failure_exception => e
     lifecycle.update_test_step do |step|
       step.status = ResultUtils.status(e)
       step.status_details = ResultUtils.status_details(e)

allure-ruby-commons/lib/allure_ruby_commons/config.rb

@@ -2,6 +2,7 @@
 
 require "logger"
 require "singleton"
+require "rspec/expectations"
 
 module Allure
   # Allure configuration class
@@ -11,7 +12,7 @@ class Config
     # @return [Array<String>] valid log levels
     LOGLEVELS = %w[DEBUG INFO WARN ERROR FATAL UNKNOWN].freeze
 
-    attr_writer :environment, :logger
+    attr_writer :environment, :logger, :failure_exception
 
     attr_accessor :results_directory,
                   :logging_level,
@@ -41,5 +42,12 @@ def environment
     def logger
       @logger ||= Logger.new($stdout, level: logging_level)
     end
+
+    # Exception class that corresponds to test failure
+    #
+    # @return [Class]
+    def failure_exception
+      @failure_exception ||= RSpec::Expectations::ExpectationNotMetError
+    end
   end
 end

allure-ruby-commons/lib/allure_ruby_commons/result_utils.rb

@@ -152,7 +152,7 @@ def issue_link(name, value, link_pattern)
       # @param [Exception] exception
       # @return [Symbol]
       def status(exception)
-        exception.is_a?(RSpec::Expectations::ExpectationNotMetError) ? Status::FAILED : Status::BROKEN
+        exception.is_a?(Allure.configuration.failure_exception) ? Status::FAILED : Status::BROKEN
       end
 
       # Get exception status detail