Skip to content

Commit

Permalink
Merge pull request #268 from creative-commoners/pulls/5/rerun
Browse files Browse the repository at this point in the history
NEW Rerun failed features in ci
GuySartorelli authored Apr 12, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 8bd1452 + e03d426 commit f577e7e
Showing 4 changed files with 419 additions and 0 deletions.
1 change: 1 addition & 0 deletions config/silverstripe.yml
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ parameters:
silverstripe_extension.ajax_steps: ~
silverstripe_extension.ajax_timeout: ~
silverstripe_extension.admin_url: ~
silverstripe_extension.is_ci: ~
silverstripe_extension.login_url: ~
silverstripe_extension.screenshot_path: ~
silverstripe_extension.module:
22 changes: 22 additions & 0 deletions src/Extension.php
Original file line number Diff line number Diff line change
@@ -19,6 +19,9 @@
use Symfony\Component\Console\Input\ArgvInput;
use Symfony\Component\DependencyInjection\Definition;
use Symfony\Component\DependencyInjection\Reference;
use Behat\Behat\Tester\ServiceContainer\TesterExtension;
use SilverStripe\BehatExtension\Utility\RerunTotalStatistics;
use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester;

/*
* This file is part of the SilverStripe\BehatExtension
@@ -100,6 +103,22 @@ public function load(ContainerBuilder $container, array $config)
$container->setParameter('silverstripe_extension.region_map', $config['region_map']);
}
$container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
$container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);

// When running in CI, behat scenarios will occasionally sporadically fail
// Replaces services with custom implementations that will rerun failed features
// Note that features rather than scenarios need to be rerun to ensure that
// everything is setup and torn down correctly and that "Background" bits of
// feature fits are rerun
if ($config['is_ci']) {
$definition = new Definition(RerunRuntimeSuiteTester::class, array(
new Reference(TesterExtension::SPECIFICATION_TESTER_ID)
));
$container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition);

$definition = new Definition(RerunTotalStatistics::class);
$container->setDefinition('output.pretty.statistics', $definition);
}
}

/**
@@ -144,6 +163,9 @@ public function configure(ArrayNodeDefinition $builder)
info('Number of seconds that @retry tags will retry for')->
defaultValue(2)->
end()->
scalarNode('is_ci')->
defaultValue(false)->
end()->
arrayNode('ajax_steps')->
defaultValue(array(
'go to',
82 changes: 82 additions & 0 deletions src/Utility/RerunRuntimeSuiteTester.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
<?php

namespace SilverStripe\BehatExtension\Utility;

use Behat\Testwork\Environment\Environment;
use Behat\Testwork\Specification\SpecificationIterator;
use Behat\Testwork\Tester\Result\IntegerTestResult;
use Behat\Testwork\Tester\Result\TestResult;
use Behat\Testwork\Tester\Result\TestResults;
use Behat\Testwork\Tester\Result\TestWithSetupResult;
use Behat\Testwork\Tester\Setup\SuccessfulSetup;
use Behat\Testwork\Tester\Setup\SuccessfulTeardown;
use Behat\Testwork\Tester\SpecificationTester;
use Behat\Testwork\Tester\SuiteTester;

/**
* Copy paste of Behat\Testwork\Tester\Runtime\RuntimeSuiteTester which is a final class
*
* Modified so that it reruns failed features
*/
class RerunRuntimeSuiteTester implements SuiteTester
{
/**
* @var SpecificationTester
*/
private $specTester;

/**
* Initializes tester.
*
* @param SpecificationTester $specTester
*/
public function __construct(SpecificationTester $specTester)
{
$this->specTester = $specTester;
}

/**
* {@inheritdoc}
*/
public function setUp(Environment $env, SpecificationIterator $iterator, $skip)
{
return new SuccessfulSetup();
}

/**
* {@inheritdoc}
*/
public function test(Environment $env, SpecificationIterator $iterator, $skip = false)
{
$results = array();
foreach ($iterator as $specification) {
$setup = $this->specTester->setUp($env, $specification, $skip);
$localSkip = !$setup->isSuccessful() || $skip;
$testResult = $this->specTester->test($env, $specification, $localSkip);
$teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);

// start modifications here
if (!$testResult->isPassed()) {
file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL);
$setup = $this->specTester->setUp($env, $specification, $skip);
$localSkip = !$setup->isSuccessful() || $skip;
$testResult = $this->specTester->test($env, $specification, $localSkip);
$teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
}
// end modifications here

$integerResult = new IntegerTestResult($testResult->getResultCode());
$results[] = new TestWithSetupResult($setup, $integerResult, $teardown);
}

return new TestResults($results);
}

/**
* {@inheritdoc}
*/
public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result)
{
return new SuccessfulTeardown();
}
}
314 changes: 314 additions & 0 deletions src/Utility/RerunTotalStatistics.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,314 @@
<?php

namespace SilverStripe\BehatExtension\Utility;

use Behat\Behat\Tester\Result\StepResult;
use Behat\Testwork\Counter\Memory;
use Behat\Testwork\Counter\Timer;
use Behat\Testwork\Tester\Result\TestResult;
use Behat\Testwork\Tester\Result\TestResults;
use Behat\Behat\Output\Statistics\Statistics;
use Behat\Behat\Output\Statistics\ScenarioStat;
use Behat\Behat\Output\Statistics\StepStat;
use Behat\Behat\Output\Statistics\HookStat;

/**
* Copy paste of Behat\Behat\Output\Statistics\TotalStatistics which is a final class
*
* Modified to remove duplicated stats from reruns
*/
class RerunTotalStatistics implements Statistics
{
/**
* @var Timer
*/
private $timer;
/**
* @var Memory
*/
private $memory;
/**
* @var array
*/
private $scenarioCounters = array();
/**
* @var array
*/
private $stepCounters = array();
/**
* @var ScenarioStat[]
*/
private $failedScenarioStats = array();
/**
* @var ScenarioStat[]
*/
private $skippedScenarioStats = array();
/**
* @var StepStat[]
*/
private $failedStepStats = array();
/**
* @var StepStat[]
*/
private $pendingStepStats = array();
/**
* @var HookStat[]
*/
private $failedHookStats = array();
// start modifications here
/**
* @var StepStat[]
*/
private $passedStepStats = array();
// end modifications here

/**
* Initializes statistics.
*/
public function __construct()
{
$this->resetAllCounters();

$this->timer = new Timer();
$this->memory = new Memory();
}

public function resetAllCounters()
{
$this->scenarioCounters = $this->stepCounters = array(
TestResult::PASSED => 0,
TestResult::FAILED => 0,
StepResult::UNDEFINED => 0,
TestResult::PENDING => 0,
TestResult::SKIPPED => 0
);
}

/**
* Starts timer.
*/
public function startTimer()
{
$this->timer->start();
}

/**
* Stops timer.
*/
public function stopTimer()
{
$this->timer->stop();
}

/**
* Returns timer object.
*
* @return Timer
*/
public function getTimer()
{
return $this->timer;
}

/**
* Returns memory usage object.
*
* @return Memory
*/
public function getMemory()
{
return $this->memory;
}

/**
* Registers scenario stat.
*
* @param ScenarioStat $stat
*/
public function registerScenarioStat(ScenarioStat $stat)
{
if (TestResults::NO_TESTS === $stat->getResultCode()) {
return;
}

$this->scenarioCounters[$stat->getResultCode()]++;

// start modifications here
if (TestResult::FAILED === $stat->getResultCode()) {
// Ensure that any scenario reruns aren't counted as additional failures
$alreadyHasFailure = false;
foreach ($this->failedScenarioStats as $failedStat) {
if ($failedStat->getPath() === $stat->getPath()) {
$alreadyHasFailure = true;
break;
}
}
if (!$alreadyHasFailure) {
$this->failedScenarioStats[] = $stat;
} else {
$this->scenarioCounters[TestResult::FAILED]--;
}
}

if (TestResult::PASSED == $stat->getResultCode()) {
// Remove the scenario from the failed scenarios list if it passes on rerun
$newFailedScenarioStats = [];
foreach ($this->failedScenarioStats as $failedStat) {
if ($failedStat->getPath() !== $stat->getPath()) {
$newFailedScenarioStats[] = $failedStat;
} else {
$this->scenarioCounters[TestResult::FAILED]--;
}
}
$this->failedScenarioStats = $newFailedScenarioStats;
}
// end modifications here

if (TestResult::SKIPPED === $stat->getResultCode()) {
$this->skippedScenarioStats[] = $stat;
}
}

/**
* Registers step stat.
*
* @param StepStat $stat
*/
public function registerStepStat(StepStat $stat)
{
$this->stepCounters[$stat->getResultCode()]++;

// start modifications here
if (TestResult::FAILED === $stat->getResultCode()) {
// Ensure that any scenario reruns don't double count step failures
$alreadyHasFailure = false;
foreach ($this->failedStepStats as $failedStat) {
if ($failedStat->getPath() === $stat->getPath()) {
$alreadyHasFailure = true;
break;
}
}
if (!$alreadyHasFailure) {
$this->failedStepStats[] = $stat;
} else {
$this->stepCounters[TestResult::FAILED]--;
}
}

if (TestResult::PASSED == $stat->getResultCode()) {
// Remove any duplicate passes on scenario rerun
$alreadyHasSuccess = false;
foreach ($this->passedStepStats as $passedStat) {
if ($passedStat->getPath() === $stat->getPath()) {
$alreadyHasSuccess = true;
break;
}
}
if (!$alreadyHasSuccess) {
$this->passedStepStats[] = $stat;
} else {
$this->stepCounters[TestResult::PASSED]--;
}

// Remove the step from the failed steps list if it passes on scenario rerun
$newFailedStepStats = [];
foreach ($this->failedStepStats as $failedStat) {
if ($failedStat->getPath() !== $stat->getPath()) {
$newFailedStepStats[] = $failedStat;
} else {
$this->stepCounters[TestResult::FAILED]--;
}
}
$this->failedStepStats = $newFailedStepStats;
}
// end modifications here

if (TestResult::PENDING === $stat->getResultCode()) {
$this->pendingStepStats[] = $stat;
}
}

/**
* Registers hook stat.
*
* @param HookStat $stat
*/
public function registerHookStat(HookStat $stat)
{
if ($stat->isSuccessful()) {
return;
}

$this->failedHookStats[] = $stat;
}

/**
* Returns counters for different scenario result codes.
*
* @return array[]
*/
public function getScenarioStatCounts()
{
return $this->scenarioCounters;
}

/**
* Returns skipped scenario stats.
*
* @return ScenarioStat[]
*/
public function getSkippedScenarios()
{
return $this->skippedScenarioStats;
}

/**
* Returns failed scenario stats.
*
* @return ScenarioStat[]
*/
public function getFailedScenarios()
{
return $this->failedScenarioStats;
}

/**
* Returns counters for different step result codes.
*
* @return array[]
*/
public function getStepStatCounts()
{
return $this->stepCounters;
}

/**
* Returns failed step stats.
*
* @return StepStat[]
*/
public function getFailedSteps()
{
return $this->failedStepStats;
}

/**
* Returns pending step stats.
*
* @return StepStat[]
*/
public function getPendingSteps()
{
return $this->pendingStepStats;
}

/**
* Returns failed hook stats.
*
* @return HookStat[]
*/
public function getFailedHookStats()
{
return $this->failedHookStats;
}
}

0 comments on commit f577e7e

Please sign in to comment.