From 789b22bdb55089d24e4aea1ef1d19d8715f6495a Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Thu, 5 Nov 2015 16:52:42 +0100 Subject: [PATCH 1/2] Add test case for #857 --- .../test/retryAnalyzer/RetryAnalyzerTest.java | 25 +++++++++ .../github857/GitHub857Listener.java | 54 +++++++++++++++++++ .../github857/GitHub857Retry.java | 19 +++++++ .../github857/GitHub857Sample.java | 15 ++++++ 4 files changed, 113 insertions(+) create mode 100644 src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java create mode 100644 src/test/java/test/retryAnalyzer/github857/GitHub857Retry.java create mode 100644 src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java diff --git a/src/test/java/test/retryAnalyzer/RetryAnalyzerTest.java b/src/test/java/test/retryAnalyzer/RetryAnalyzerTest.java index ee3fca6dad..b43e632069 100644 --- a/src/test/java/test/retryAnalyzer/RetryAnalyzerTest.java +++ b/src/test/java/test/retryAnalyzer/RetryAnalyzerTest.java @@ -12,6 +12,8 @@ import org.testng.annotations.Test; import test.SimpleBaseTest; +import test.retryAnalyzer.github857.GitHub857Listener; +import test.retryAnalyzer.github857.GitHub857Sample; public class RetryAnalyzerTest extends SimpleBaseTest { @Test @@ -34,4 +36,27 @@ public void testInvocationCounts() { List skipped = tla.getSkippedTests(); assertEquals(skipped.size(), InvocationCountTest.invocations.size() - fsp.size()); } + + @Test(description = "GITHUB-857: onTestFailure not being called when test is retried") + public void onTestFailureShouldBeCalledWhenTestIsRetried() { + TestNG tng = create(GitHub857Sample.class); + TestListenerAdapter tla = new TestListenerAdapter(); + tng.addListener(tla); + + tng.run(); + + assertEquals(tla.getPassedTests().size(), 0); + assertEquals(tla.getFailedButWithinSuccessPercentageTests().size(), 0); + assertEquals(tla.getSkippedTests().size(), 1); + assertEquals(tla.getSkippedTests().get(0).getMethod().getMethodName(), "test"); + assertEquals(tla.getFailedTests().size(), 1); + assertEquals(tla.getFailedTests().get(0).getMethod().getMethodName(), "test"); + + assertEquals(GitHub857Listener.passedTests.size(), 0); + assertEquals(GitHub857Listener.failedButWSPerTests.size(), 0); + assertEquals(GitHub857Listener.skippedTests.size(), 1); + assertEquals(GitHub857Listener.skippedTests.get(0).getMethod().getMethodName(), "test"); + assertEquals(GitHub857Listener.failedTests.size(), 1); + assertEquals(GitHub857Listener.failedTests.get(0).getMethod().getMethodName(), "test"); + } } diff --git a/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java b/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java new file mode 100644 index 0000000000..f40b5d2962 --- /dev/null +++ b/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java @@ -0,0 +1,54 @@ +package test.retryAnalyzer.github857; + +import org.testng.ITestContext; +import org.testng.ITestResult; +import org.testng.TestListenerAdapter; +import org.testng.collections.Lists; + +import java.util.Collections; +import java.util.List; + +public class GitHub857Listener extends TestListenerAdapter { + + public static List + passedTests = + Collections.synchronizedList(Lists.newArrayList()); + public static List + failedTests = + Collections.synchronizedList(Lists.newArrayList()); + public static List + skippedTests = + Collections.synchronizedList(Lists.newArrayList()); + public static List + failedButWSPerTests = + Collections.synchronizedList(Lists.newArrayList()); + + @Override + public void onTestSuccess(ITestResult result) { + passedTests.add(result); + } + + @Override + public void onTestFailure(ITestResult result) { + failedTests.add(result); + } + + @Override + public void onTestSkipped(ITestResult result) { + skippedTests.add(result); + } + + @Override + public void onTestFailedButWithinSuccessPercentage(ITestResult result) { + failedButWSPerTests.add(result); + } + + @Override + public void onTestStart(ITestResult result) {} + + @Override + public void onStart(ITestContext context) {} + + @Override + public void onFinish(ITestContext context) {} +} diff --git a/src/test/java/test/retryAnalyzer/github857/GitHub857Retry.java b/src/test/java/test/retryAnalyzer/github857/GitHub857Retry.java new file mode 100644 index 0000000000..74a51b7e4a --- /dev/null +++ b/src/test/java/test/retryAnalyzer/github857/GitHub857Retry.java @@ -0,0 +1,19 @@ +package test.retryAnalyzer.github857; + +import org.testng.IRetryAnalyzer; +import org.testng.ITestResult; + +public class GitHub857Retry implements IRetryAnalyzer { + + private static final int MAX_RETRY_COUNT_PER_TEST = 1; + private int retryCountPerTest = 0; + + @Override + public boolean retry(ITestResult result) { + if (retryCountPerTest < MAX_RETRY_COUNT_PER_TEST) { + retryCountPerTest++; + return true; + } + return false; + } +} diff --git a/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java b/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java new file mode 100644 index 0000000000..e655969a42 --- /dev/null +++ b/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java @@ -0,0 +1,15 @@ +package test.retryAnalyzer.github857; + +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +import static org.testng.Assert.fail; + +@Listeners(GitHub857Listener.class) +public class GitHub857Sample { + + @Test(retryAnalyzer = GitHub857Retry.class) + public void test() { + fail("Failing"); + } +} From 43847cb8f66e137d4aa245d05c548fe2c9f833ea Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Fri, 6 Nov 2015 13:04:13 +0100 Subject: [PATCH 2/2] Update sample after comments --- .../retryAnalyzer/github857/GitHub857Listener.java | 6 +++++- .../retryAnalyzer/github857/GitHub857Sample.java | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java b/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java index f40b5d2962..2c3714b9d1 100644 --- a/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java +++ b/src/test/java/test/retryAnalyzer/github857/GitHub857Listener.java @@ -30,11 +30,13 @@ public void onTestSuccess(ITestResult result) { @Override public void onTestFailure(ITestResult result) { + System.out.println("FAILURE"); failedTests.add(result); } @Override public void onTestSkipped(ITestResult result) { + System.out.println("SKIPPED"); skippedTests.add(result); } @@ -44,7 +46,9 @@ public void onTestFailedButWithinSuccessPercentage(ITestResult result) { } @Override - public void onTestStart(ITestResult result) {} + public void onTestStart(ITestResult result) { + System.out.println("START"); + } @Override public void onStart(ITestContext context) {} diff --git a/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java b/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java index e655969a42..d15f217e08 100644 --- a/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java +++ b/src/test/java/test/retryAnalyzer/github857/GitHub857Sample.java @@ -1,5 +1,8 @@ package test.retryAnalyzer.github857; +import org.testng.ITestContext; +import org.testng.ITestNGMethod; +import org.testng.annotations.BeforeTest; import org.testng.annotations.Listeners; import org.testng.annotations.Test; @@ -8,7 +11,14 @@ @Listeners(GitHub857Listener.class) public class GitHub857Sample { - @Test(retryAnalyzer = GitHub857Retry.class) + @BeforeTest(alwaysRun = true) + public void beforeTest(ITestContext context) { + for (ITestNGMethod method : context.getAllTestMethods()) { + method.setRetryAnalyzer(new GitHub857Retry()); + } + } + + @Test public void test() { fail("Failing"); }