-
Notifications
You must be signed in to change notification settings - Fork 402
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
new ReducingMethod KEEP_ONLY_LATEST_SCENARIO_RUNS #914
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,9 @@ | |
import com.fasterxml.jackson.databind.JsonMappingException; | ||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; | ||
import com.google.common.collect.Lists; | ||
|
||
import net.masterthought.cucumber.json.Element; | ||
import net.masterthought.cucumber.json.Feature; | ||
import net.masterthought.cucumber.reducers.ReducingMethod; | ||
import org.apache.commons.configuration.ConfigurationException; | ||
|
@@ -22,6 +25,8 @@ | |
import java.util.Arrays; | ||
import java.util.Iterator; | ||
import java.util.List; | ||
import java.util.ListIterator; | ||
import java.util.Optional; | ||
import java.util.logging.Level; | ||
import java.util.logging.Logger; | ||
|
||
|
@@ -71,6 +76,9 @@ public List<Feature> parseJsonFiles(List<String> jsonFiles) { | |
continue; | ||
} | ||
Feature[] features = parseForFeature(jsonFile); | ||
if (configuration.containsReducingMethod(ReducingMethod.KEEP_ONLY_LATEST_SCENARIO_RUNS)) { | ||
keepOnlyLatestScenarioRuns(features); | ||
} | ||
LOG.log(Level.INFO, String.format("File '%s' contains %d features", jsonFile, features.length)); | ||
featureResults.addAll(Arrays.asList(features)); | ||
} | ||
|
@@ -82,6 +90,42 @@ public List<Feature> parseJsonFiles(List<String> jsonFiles) { | |
|
||
return featureResults; | ||
} | ||
|
||
/** | ||
* If the JSON file has the same scenarios run multiple times, keep only the | ||
* latest scenario's run. | ||
*/ | ||
private void keepOnlyLatestScenarioRuns(Feature[] features) { | ||
for (Feature feature : features) { | ||
Element[] elements = feature.getElements(); | ||
List<Element> elementList = Lists.newArrayList(elements); | ||
ListIterator<Element> li = elementList.listIterator(elementList.size()); | ||
Optional<Element> lastElement = Optional.empty(); | ||
int numRemoved = 0; | ||
while (li.hasPrevious()) { | ||
Element element = li.previous(); | ||
if (lastElement.isPresent() && element.getId().equals(lastElement.get().getId())) { | ||
if (LOG.isLoggable(Level.FINE)) { | ||
LOG.log(Level.FINE, String.format("Reducing method KEEP_ONLY_EARLIEST_SCENARIO_RUNS is removing an earlier test result of scenario %s", feature.getName())); | ||
} | ||
li.remove(); | ||
++numRemoved; | ||
} else { | ||
addRetryNumberToElmIfNeeded(lastElement, numRemoved); | ||
lastElement = Optional.of(element); | ||
numRemoved = 0; | ||
} | ||
} | ||
addRetryNumberToElmIfNeeded(lastElement, numRemoved); | ||
feature.setElements(elementList.toArray(new Element[0])); | ||
} | ||
} | ||
|
||
private void addRetryNumberToElmIfNeeded(Optional<Element> lastElement, int numRemoved) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is Java, we don't shorten names to |
||
if (lastElement.isPresent() && numRemoved > 0) { | ||
lastElement.get().appendToName(" [Retry count " + (numRemoved + 1) + "]"); | ||
} | ||
} | ||
|
||
/** | ||
* Reads passed file and returns parsed features. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,7 +15,7 @@ public class Element implements Durationable { | |
|
||
// Start: attributes from JSON file report | ||
private final String id = null; | ||
private final String name = null; | ||
private String name = null; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. name is immutable There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i change this to mutable because i use the Name in the report because I append " (${numRetries)" so that it shows up in the report. alternatively we could create a new element in the html/xslt or whatever. |
||
private final String type = null; | ||
private final String description = null; | ||
private final String keyword = null; | ||
|
@@ -157,4 +157,12 @@ private void calculateDuration() { | |
duration += step.getResult().getDuration(); | ||
} | ||
} | ||
|
||
/** | ||
* Append a string to the name. | ||
* @param str The string to append. | ||
*/ | ||
public void appendToName(String str) { | ||
name += str; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
package net.masterthought.cucumber; | ||
|
||
import java.io.File; | ||
import java.io.IOException; | ||
import java.util.ArrayList; | ||
import java.util.Collections; | ||
import java.util.List; | ||
import java.util.UUID; | ||
|
||
import org.apache.commons.io.FileUtils; | ||
import org.junit.After; | ||
import org.junit.Assert; | ||
import org.junit.Before; | ||
import org.junit.Test; | ||
|
||
import net.masterthought.cucumber.json.support.Status; | ||
import net.masterthought.cucumber.presentation.PresentationMode; | ||
import net.masterthought.cucumber.reducers.ReducingMethod; | ||
|
||
public class RemoveFailuresDueToRetriesTest { | ||
private static final String WITH_RETRIES_JSON = "with-retries.json"; | ||
|
||
File reportDir; | ||
File jsonFile; | ||
|
||
@Before | ||
public void before() throws IOException { | ||
File target = new File("target"); | ||
reportDir = new File(target, UUID.randomUUID().toString()); | ||
reportDir.mkdirs(); | ||
jsonFile = new File(reportDir, WITH_RETRIES_JSON); | ||
|
||
FileUtils.copyInputStreamToFile(this.getClass().getResourceAsStream("/json/" + WITH_RETRIES_JSON), jsonFile); | ||
} | ||
|
||
@After | ||
public void after() throws IOException { | ||
FileUtils.deleteDirectory(reportDir); | ||
} | ||
|
||
@Test | ||
public void testRetryRemoval() throws IOException { | ||
|
||
File reportOutputDirectory = reportDir; | ||
List<String> jsonFiles = new ArrayList<>(); | ||
jsonFiles.add(jsonFile.getAbsolutePath()); | ||
|
||
String buildNumber = "1"; | ||
String projectName = "cucumberProject"; | ||
|
||
Configuration configuration = new Configuration(reportOutputDirectory, projectName); | ||
// optional configuration - check javadoc for details | ||
configuration.addPresentationModes(PresentationMode.RUN_WITH_JENKINS); | ||
// do not make scenario failed when step has status SKIPPED | ||
configuration.setNotFailingStatuses(Collections.singleton(Status.SKIPPED)); | ||
configuration.setBuildNumber(buildNumber); | ||
// addidtional metadata presented on main page | ||
configuration.addClassifications("Platform", "Windows"); | ||
configuration.addClassifications("Browser", "Google Chrome"); | ||
configuration.addClassifications("Branch", "release/1.0"); | ||
configuration.addReducingMethod(ReducingMethod.KEEP_ONLY_LATEST_SCENARIO_RUNS); | ||
|
||
|
||
ReportBuilder reportBuilder = new ReportBuilder(jsonFiles, configuration); | ||
Reportable result = reportBuilder.generateReports(); | ||
|
||
Assert.assertEquals("Should not report the retried steps as failures!", 0, result.getFailedSteps()); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
AFAIR JSON does not have results for multiple times. For that case you would rather have more than one JSON file. Can you explain this case?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
also looks similar to https://github.com/damianszczepanik/cucumber-reporting/pull/894/files#diff-858efbefb85a2d5a42faf0d6a15da70bR64
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@damianszczepanik - I know this is an old PR. This case happens when we use
TestNGCucumberRunner
and use TestNG's retry analyzer capability to retry tests to disregard flaky tests.Assuming that we run 1 test with re-run enabled -
What this does is, in case of flaky behavior, where a test would fail first and then would pass when re-tried - it creates 2 entries in JSON for the same test - 1 for the failed test and 2nd for the same test, re-tried. So the final report shows that there were 2 tests.
If the re-tried scenario also fails, it is counted as another failed test. so total 2 failed tests
If the re-tried scenario passed, the report says, 1 failed and 1 passed.
This PR seems to solve that problem.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Exactly. Sorry I didn't respond earlier, got moved on to other priorities and never had a chance to come back and answer the questions/fixes.