Skip to content

Commit ab300f8

Browse files
katcharovkatcharov
authored
Retry flaky unified tests (#1565)
JAVA-5795
1 parent 42b1a37 commit ab300f8

File tree

10 files changed

+238
-54
lines changed

10 files changed

+238
-54
lines changed

‎driver-core/src/main/com/mongodb/assertions/Assertions.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,19 @@ public static boolean assertTrue(final boolean value) throws AssertionError {
179179
return true;
180180
}
181181

182+
/**
183+
* @param value A value to check.
184+
* @param message The message.
185+
* @return {@code true}.
186+
* @throws AssertionError If {@code value} is {@code false}.
187+
*/
188+
public static boolean assertTrue(final boolean value, final String message) throws AssertionError {
189+
if (!value) {
190+
throw new AssertionError(message);
191+
}
192+
return true;
193+
}
194+
182195
/**
183196
* @param value A value to check.
184197
* @return {@code false}.

‎driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", true)
2828
}
2929
}
3030
}

‎driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", false)
2828
}
2929
}
3030
}

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c
9999
@MethodSource("data")
100100
@Override
101101
public void shouldPassAllOutcomes(
102+
final String testName,
102103
@Nullable final String fileDescription,
103104
@Nullable final String testDescription,
104105
@Nullable final String directoryName,
106+
final int attemptNumber,
107+
final int totalAttempts,
105108
final String schemaVersion,
106109
@Nullable final BsonArray runOnRequirements,
107110
final BsonArray entitiesArray,
108111
final BsonArray initialData,
109112
final BsonDocument definition) {
110113
try {
111-
super.shouldPassAllOutcomes(fileDescription,
114+
super.shouldPassAllOutcomes(
115+
testName,
116+
fileDescription,
112117
testDescription,
113118
directoryName,
119+
attemptNumber,
120+
totalAttempts,
114121
schemaVersion,
115122
runOnRequirements,
116123
entitiesArray,

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,19 @@
2424
import com.mongodb.client.unified.UnifiedTest;
2525
import com.mongodb.client.unified.UnifiedTestModifications;
2626
import com.mongodb.client.vault.ClientEncryption;
27+
import com.mongodb.lang.NonNull;
2728
import com.mongodb.reactivestreams.client.MongoClients;
2829
import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets;
2930
import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl;
3031
import com.mongodb.reactivestreams.client.syncadapter.SyncClientEncryption;
3132
import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket;
3233
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient;
3334
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase;
35+
import org.junit.jupiter.params.provider.Arguments;
36+
37+
import java.io.IOException;
38+
import java.net.URISyntaxException;
39+
import java.util.Collection;
3440

3541
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier;
3642
import static com.mongodb.client.unified.UnifiedTestModifications.TestDef;
@@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) {
94100
disableSleep();
95101
}
96102
}
103+
104+
@NonNull
105+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
106+
return getTestData(directory, true);
107+
}
97108
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525
import com.mongodb.client.gridfs.GridFSBuckets;
2626
import com.mongodb.client.internal.ClientEncryptionImpl;
2727
import com.mongodb.client.vault.ClientEncryption;
28+
import com.mongodb.lang.NonNull;
29+
import org.junit.jupiter.params.provider.Arguments;
30+
31+
import java.io.IOException;
32+
import java.net.URISyntaxException;
33+
import java.util.Collection;
2834

2935
public abstract class UnifiedSyncTest extends UnifiedTest {
3036
protected UnifiedSyncTest() {
@@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) {
4450
protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) {
4551
return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings);
4652
}
53+
54+
@NonNull
55+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
56+
return getTestData(directory, false);
57+
}
4758
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java‎

Lines changed: 102 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,11 @@
6262
import java.io.File;
6363
import java.io.IOException;
6464
import java.net.URISyntaxException;
65+
import java.text.MessageFormat;
6566
import java.util.ArrayList;
6667
import java.util.Collection;
6768
import java.util.Collections;
69+
import java.util.HashSet;
6870
import java.util.List;
6971
import java.util.Set;
7072
import java.util.concurrent.ExecutionException;
@@ -81,6 +83,8 @@
8183
import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime;
8284
import static com.mongodb.client.test.CollectionHelper.killAllSessions;
8385
import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet;
86+
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier;
87+
import static com.mongodb.client.unified.UnifiedTestModifications.applyCustomizations;
8488
import static com.mongodb.client.unified.UnifiedTestModifications.testDef;
8589
import static java.util.Collections.singletonList;
8690
import static java.util.stream.Collectors.toList;
@@ -91,6 +95,7 @@
9195
import static org.junit.jupiter.api.Assertions.assertNull;
9296
import static org.junit.jupiter.api.Assertions.assertTrue;
9397
import static org.junit.jupiter.api.Assertions.fail;
98+
import static org.junit.jupiter.api.Assumptions.abort;
9499
import static org.junit.jupiter.api.Assumptions.assumeFalse;
95100
import static org.junit.jupiter.api.Assumptions.assumeTrue;
96101
import static util.JsonPoweredTestHelper.getTestDocument;
@@ -101,6 +106,10 @@ public abstract class UnifiedTest {
101106
private static final Set<String> PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton(
102107
"wait queue timeout errors include details about checked out connections");
103108

109+
public static final int RETRY_ATTEMPTS = 3;
110+
public static final int FORCE_FLAKY_ATTEMPTS = 10;
111+
private static final Set<String> ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE = new HashSet<>();
112+
104113
@Nullable
105114
private String fileDescription;
106115
private String schemaVersion;
@@ -156,32 +165,49 @@ public Entities getEntities() {
156165
}
157166

158167
@NonNull
159-
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
168+
protected static Collection<Arguments> getTestData(final String directory, final boolean isReactive)
169+
throws URISyntaxException, IOException {
160170
List<Arguments> data = new ArrayList<>();
161171
for (File file : getTestFiles("/" + directory + "/")) {
162172
BsonDocument fileDocument = getTestDocument(file);
163-
164173
for (BsonValue cur : fileDocument.getArray("tests")) {
165-
data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument()));
174+
175+
final BsonDocument testDocument = cur.asDocument();
176+
String testDescription = testDocument.getString("description").getValue();
177+
String fileDescription = fileDocument.getString("description").getValue();
178+
TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive);
179+
applyCustomizations(testDef);
180+
181+
boolean forceFlaky = testDef.wasAssignedModifier(Modifier.FORCE_FLAKY);
182+
boolean retry = forceFlaky || testDef.wasAssignedModifier(Modifier.RETRY);
183+
184+
int attempts;
185+
if (retry) {
186+
attempts = forceFlaky ? FORCE_FLAKY_ATTEMPTS : RETRY_ATTEMPTS;
187+
} else {
188+
attempts = 1;
189+
}
190+
191+
for (int attempt = 1; attempt <= attempts; attempt++) {
192+
String testName = MessageFormat.format("{0}: {1}", fileDescription, testDescription);
193+
data.add(Arguments.of(
194+
testName,
195+
fileDescription,
196+
testDescription,
197+
directory,
198+
attempt,
199+
attempts,
200+
fileDocument.getString("schemaVersion").getValue(),
201+
fileDocument.getArray("runOnRequirements", null),
202+
fileDocument.getArray("createEntities", new BsonArray()),
203+
fileDocument.getArray("initialData", new BsonArray()),
204+
testDocument.clone()));
205+
}
166206
}
167207
}
168208
return data;
169209
}
170210

171-
@NonNull
172-
private static Arguments createTestData(
173-
final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) {
174-
return Arguments.of(
175-
fileDocument.getString("description").getValue(),
176-
testDocument.getString("description").getValue(),
177-
directory,
178-
fileDocument.getString("schemaVersion").getValue(),
179-
fileDocument.getArray("runOnRequirements", null),
180-
fileDocument.getArray("createEntities", new BsonArray()),
181-
fileDocument.getArray("initialData", new BsonArray()),
182-
testDocument);
183-
}
184-
185211
protected BsonDocument getDefinition() {
186212
return definition;
187213
}
@@ -194,9 +220,12 @@ protected BsonDocument getDefinition() {
194220

195221
@BeforeEach
196222
public void setUp(
223+
final String testName,
197224
@Nullable final String fileDescription,
198225
@Nullable final String testDescription,
199226
@Nullable final String directoryName,
227+
final int attemptNumber,
228+
final int totalAttempts,
200229
final String schemaVersion,
201230
@Nullable final BsonArray runOnRequirements,
202231
final BsonArray entitiesArray,
@@ -218,9 +247,9 @@ public void setUp(
218247
ignoreExtraEvents = false;
219248
if (directoryName != null && fileDescription != null && testDescription != null) {
220249
testDef = testDef(directoryName, fileDescription, testDescription, isReactive());
221-
UnifiedTestModifications.doSkips(testDef);
250+
applyCustomizations(testDef);
222251

223-
boolean skip = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.SKIP);
252+
boolean skip = testDef.wasAssignedModifier(Modifier.SKIP);
224253
assumeFalse(skip, "Skipping test");
225254
}
226255
skips(fileDescription, testDescription);
@@ -295,8 +324,9 @@ protected void postCleanUp(final TestDef testDef) {
295324
}
296325

297326
/**
298-
* This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)},
299-
* unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
327+
* This method is called once per
328+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless
329+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
300330
*/
301331
protected void skips(final String fileDescription, final String testDescription) {
302332
}
@@ -305,40 +335,72 @@ protected boolean isReactive() {
305335
return false;
306336
}
307337

308-
@ParameterizedTest(name = "{0}: {1}")
338+
@ParameterizedTest(name = "{0}")
309339
@MethodSource("data")
310340
public void shouldPassAllOutcomes(
341+
final String testName,
311342
@Nullable final String fileDescription,
312343
@Nullable final String testDescription,
313344
@Nullable final String directoryName,
345+
final int attemptNumber,
346+
final int totalAttempts,
314347
final String schemaVersion,
315348
@Nullable final BsonArray runOnRequirements,
316349
final BsonArray entitiesArray,
317350
final BsonArray initialData,
318351
final BsonDocument definition) {
319-
BsonArray operations = definition.getArray("operations");
320-
for (int i = 0; i < operations.size(); i++) {
321-
BsonValue cur = operations.get(i);
322-
assertOperation(rootContext, cur.asDocument(), i);
352+
boolean forceFlaky = testDef.wasAssignedModifier(Modifier.FORCE_FLAKY);
353+
if (!forceFlaky) {
354+
boolean ignoreThisTest = ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.contains(testName);
355+
assumeFalse(ignoreThisTest, "Skipping a retryable test that already succeeded");
356+
// The attempt is what counts, since a test may fail with
357+
// something like "ignored", and would not be retried.
358+
// Only failures should trigger another attempt.
359+
ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.add(testName);
323360
}
361+
try {
362+
BsonArray operations = definition.getArray("operations");
363+
for (int i = 0; i < operations.size(); i++) {
364+
BsonValue cur = operations.get(i);
365+
assertOperation(rootContext, cur.asDocument(), i);
366+
}
324367

325-
if (definition.containsKey("outcome")) {
326-
assertOutcome(rootContext);
327-
}
368+
if (definition.containsKey("outcome")) {
369+
assertOutcome(rootContext);
370+
}
328371

329-
if (definition.containsKey("expectEvents")) {
330-
compareEvents(rootContext, definition);
331-
}
372+
if (definition.containsKey("expectEvents")) {
373+
compareEvents(rootContext, definition);
374+
}
332375

333-
if (definition.containsKey("expectLogMessages")) {
334-
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
335-
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
336-
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
337-
if (getMongoClientSettings().getClusterSettings()
338-
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
339-
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
376+
if (definition.containsKey("expectLogMessages")) {
377+
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
378+
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
379+
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
380+
if (getMongoClientSettings().getClusterSettings()
381+
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
382+
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
383+
}
384+
compareLogMessages(rootContext, definition, tweaks);
385+
}
386+
} catch (TestAbortedException e) {
387+
// if a test is ignored, we do not retry
388+
throw e;
389+
} catch (Throwable e) {
390+
if (forceFlaky) {
391+
throw e;
392+
}
393+
if (testDef != null && !testDef.matchesThrowable(e)) {
394+
// if the throwable is not matched, test definitions were not intended to apply; rethrow it
395+
throw e;
340396
}
341-
compareLogMessages(rootContext, definition, tweaks);
397+
boolean isLastAttempt = attemptNumber == totalAttempts;
398+
if (isLastAttempt) {
399+
throw e;
400+
}
401+
402+
ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.remove(testName);
403+
abort("Ignoring failure and retrying attempt " + attemptNumber);
342404
}
343405
}
344406

0 commit comments

Comments
 (0)