From 1452c35e080bc121288efc6fcae8cd8f378d8277 Mon Sep 17 00:00:00 2001 From: Steve Boyd Date: Mon, 15 Apr 2024 14:30:19 +1200 Subject: [PATCH] [Backport] Rerun failed features in ci --- config/silverstripe.yml | 1 + src/Extension.php | 22 ++ src/Utility/RerunRuntimeSuiteTester.php | 82 +++++++ src/Utility/RerunTotalStatistics.php | 314 ++++++++++++++++++++++++ 4 files changed, 419 insertions(+) create mode 100644 src/Utility/RerunRuntimeSuiteTester.php create mode 100644 src/Utility/RerunTotalStatistics.php diff --git a/config/silverstripe.yml b/config/silverstripe.yml index 41a4e175..a7c0797d 100644 --- a/config/silverstripe.yml +++ b/config/silverstripe.yml @@ -8,6 +8,7 @@ parameters: silverstripe_extension.ajax_steps: ~ silverstripe_extension.ajax_timeout: ~ silverstripe_extension.admin_url: ~ + silverstripe_extension.is_ci: ~ silverstripe_extension.login_url: ~ silverstripe_extension.screenshot_path: ~ silverstripe_extension.module: diff --git a/src/Extension.php b/src/Extension.php index 7658b2b4..51324592 100644 --- a/src/Extension.php +++ b/src/Extension.php @@ -18,6 +18,9 @@ use RuntimeException; use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\Reference; +use Behat\Behat\Tester\ServiceContainer\TesterExtension; +use SilverStripe\BehatExtension\Utility\RerunTotalStatistics; +use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester; /* * This file is part of the SilverStripe\BehatExtension @@ -98,6 +101,22 @@ public function load(ContainerBuilder $container, array $config) $container->setParameter('silverstripe_extension.region_map', $config['region_map']); } $container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']); + $container->setParameter('silverstripe_extension.is_ci', $config['is_ci']); + + // When running in CI, behat scenarios will occasionally sporadically fail + // Replaces services with custom implementations that will rerun failed features + // Note that features rather than scenarios need to be rerun to ensure that + // everything is setup and torn down correctly and that "Background" bits of + // feature fits are rerun + if ($config['is_ci']) { + $definition = new Definition(RerunRuntimeSuiteTester::class, array( + new Reference(TesterExtension::SPECIFICATION_TESTER_ID) + )); + $container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition); + + $definition = new Definition(RerunTotalStatistics::class); + $container->setDefinition('output.pretty.statistics', $definition); + } } /** @@ -140,6 +159,9 @@ public function configure(ArrayNodeDefinition $builder) info('Number of seconds that @retry tags will retry for')-> defaultValue(2)-> end()-> + scalarNode('is_ci')-> + defaultValue(false)-> + end()-> arrayNode('ajax_steps')-> defaultValue(array( 'go to', diff --git a/src/Utility/RerunRuntimeSuiteTester.php b/src/Utility/RerunRuntimeSuiteTester.php new file mode 100644 index 00000000..630f4ecd --- /dev/null +++ b/src/Utility/RerunRuntimeSuiteTester.php @@ -0,0 +1,82 @@ +specTester = $specTester; + } + + /** + * {@inheritdoc} + */ + public function setUp(Environment $env, SpecificationIterator $iterator, $skip) + { + return new SuccessfulSetup(); + } + + /** + * {@inheritdoc} + */ + public function test(Environment $env, SpecificationIterator $iterator, $skip = false) + { + $results = array(); + foreach ($iterator as $specification) { + $setup = $this->specTester->setUp($env, $specification, $skip); + $localSkip = !$setup->isSuccessful() || $skip; + $testResult = $this->specTester->test($env, $specification, $localSkip); + $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult); + + // start modifications here + if (!$testResult->isPassed()) { + file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL); + $setup = $this->specTester->setUp($env, $specification, $skip); + $localSkip = !$setup->isSuccessful() || $skip; + $testResult = $this->specTester->test($env, $specification, $localSkip); + $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult); + } + // end modifications here + + $integerResult = new IntegerTestResult($testResult->getResultCode()); + $results[] = new TestWithSetupResult($setup, $integerResult, $teardown); + } + + return new TestResults($results); + } + + /** + * {@inheritdoc} + */ + public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result) + { + return new SuccessfulTeardown(); + } +} diff --git a/src/Utility/RerunTotalStatistics.php b/src/Utility/RerunTotalStatistics.php new file mode 100644 index 00000000..7ebc4496 --- /dev/null +++ b/src/Utility/RerunTotalStatistics.php @@ -0,0 +1,314 @@ +resetAllCounters(); + + $this->timer = new Timer(); + $this->memory = new Memory(); + } + + public function resetAllCounters() + { + $this->scenarioCounters = $this->stepCounters = array( + TestResult::PASSED => 0, + TestResult::FAILED => 0, + StepResult::UNDEFINED => 0, + TestResult::PENDING => 0, + TestResult::SKIPPED => 0 + ); + } + + /** + * Starts timer. + */ + public function startTimer() + { + $this->timer->start(); + } + + /** + * Stops timer. + */ + public function stopTimer() + { + $this->timer->stop(); + } + + /** + * Returns timer object. + * + * @return Timer + */ + public function getTimer() + { + return $this->timer; + } + + /** + * Returns memory usage object. + * + * @return Memory + */ + public function getMemory() + { + return $this->memory; + } + + /** + * Registers scenario stat. + * + * @param ScenarioStat $stat + */ + public function registerScenarioStat(ScenarioStat $stat) + { + if (TestResults::NO_TESTS === $stat->getResultCode()) { + return; + } + + $this->scenarioCounters[$stat->getResultCode()]++; + + // start modifications here + if (TestResult::FAILED === $stat->getResultCode()) { + // Ensure that any scenario reruns aren't counted as additional failures + $alreadyHasFailure = false; + foreach ($this->failedScenarioStats as $failedStat) { + if ($failedStat->getPath() === $stat->getPath()) { + $alreadyHasFailure = true; + break; + } + } + if (!$alreadyHasFailure) { + $this->failedScenarioStats[] = $stat; + } else { + $this->scenarioCounters[TestResult::FAILED]--; + } + } + + if (TestResult::PASSED == $stat->getResultCode()) { + // Remove the scenario from the failed scenarios list if it passes on rerun + $newFailedScenarioStats = []; + foreach ($this->failedScenarioStats as $failedStat) { + if ($failedStat->getPath() !== $stat->getPath()) { + $newFailedScenarioStats[] = $failedStat; + } else { + $this->scenarioCounters[TestResult::FAILED]--; + } + } + $this->failedScenarioStats = $newFailedScenarioStats; + } + // end modifications here + + if (TestResult::SKIPPED === $stat->getResultCode()) { + $this->skippedScenarioStats[] = $stat; + } + } + + /** + * Registers step stat. + * + * @param StepStat $stat + */ + public function registerStepStat(StepStat $stat) + { + $this->stepCounters[$stat->getResultCode()]++; + + // start modifications here + if (TestResult::FAILED === $stat->getResultCode()) { + // Ensure that any scenario reruns don't double count step failures + $alreadyHasFailure = false; + foreach ($this->failedStepStats as $failedStat) { + if ($failedStat->getPath() === $stat->getPath()) { + $alreadyHasFailure = true; + break; + } + } + if (!$alreadyHasFailure) { + $this->failedStepStats[] = $stat; + } else { + $this->stepCounters[TestResult::FAILED]--; + } + } + + if (TestResult::PASSED == $stat->getResultCode()) { + // Remove any duplicate passes on scenario rerun + $alreadyHasSuccess = false; + foreach ($this->passedStepStats as $passedStat) { + if ($passedStat->getPath() === $stat->getPath()) { + $alreadyHasSuccess = true; + break; + } + } + if (!$alreadyHasSuccess) { + $this->passedStepStats[] = $stat; + } else { + $this->stepCounters[TestResult::PASSED]--; + } + + // Remove the step from the failed steps list if it passes on scenario rerun + $newFailedStepStats = []; + foreach ($this->failedStepStats as $failedStat) { + if ($failedStat->getPath() !== $stat->getPath()) { + $newFailedStepStats[] = $failedStat; + } else { + $this->stepCounters[TestResult::FAILED]--; + } + } + $this->failedStepStats = $newFailedStepStats; + } + // end modifications here + + if (TestResult::PENDING === $stat->getResultCode()) { + $this->pendingStepStats[] = $stat; + } + } + + /** + * Registers hook stat. + * + * @param HookStat $stat + */ + public function registerHookStat(HookStat $stat) + { + if ($stat->isSuccessful()) { + return; + } + + $this->failedHookStats[] = $stat; + } + + /** + * Returns counters for different scenario result codes. + * + * @return array[] + */ + public function getScenarioStatCounts() + { + return $this->scenarioCounters; + } + + /** + * Returns skipped scenario stats. + * + * @return ScenarioStat[] + */ + public function getSkippedScenarios() + { + return $this->skippedScenarioStats; + } + + /** + * Returns failed scenario stats. + * + * @return ScenarioStat[] + */ + public function getFailedScenarios() + { + return $this->failedScenarioStats; + } + + /** + * Returns counters for different step result codes. + * + * @return array[] + */ + public function getStepStatCounts() + { + return $this->stepCounters; + } + + /** + * Returns failed step stats. + * + * @return StepStat[] + */ + public function getFailedSteps() + { + return $this->failedStepStats; + } + + /** + * Returns pending step stats. + * + * @return StepStat[] + */ + public function getPendingSteps() + { + return $this->pendingStepStats; + } + + /** + * Returns failed hook stats. + * + * @return HookStat[] + */ + public function getFailedHookStats() + { + return $this->failedHookStats; + } +}