RATIS-2217. Automatically re-try flaky tests in CI (#1229)

apache · Feb 26, 2025 · e604110 · e604110
1 parent 6238fe3
commit e604110
Show file tree

Hide file tree

Showing 19 changed files with 148 additions and 0 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -83,6 +83,7 @@ jobs:
           - grpc
           - server
           - misc
+          - flaky
       fail-fast: false
     uses: ./.github/workflows/check.yaml
     with:

diff --git a/dev-support/checks/unit.sh b/dev-support/checks/unit.sh
@@ -36,6 +36,10 @@ mkdir -p "$REPORT_DIR"
 export MAVEN_OPTS="-Xmx4096m"
 MAVEN_OPTIONS='-V -B'
 
+if [[ "$@" =~ "-Pflaky-tests" ]]; then
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.timeout=1200"
+fi
+
 if [[ "${FAIL_FAST}" == "true" ]]; then
   MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast -Dsurefire.skipAfterFailureCount=1"
 else

diff --git a/pom.xml b/pom.xml
@@ -222,6 +222,7 @@
     <slf4j.version>2.0.7</slf4j.version>
     <junit-bom.version>5.11.2</junit-bom.version>
     <jacoco.version>0.8.12</jacoco.version>
+    <flaky-test-groups>flaky | org.apache.ratis.test.tag.FlakyTest</flaky-test-groups>
   </properties>
 
   <dependencyManagement>
@@ -1118,6 +1119,7 @@
               <includes>
                 <include>org.apache.ratis.grpc.**</include>
               </includes>
+              <excludedGroups>${flaky-test-groups}</excludedGroups>
             </configuration>
           </plugin>
         </plugins>
@@ -1135,6 +1137,7 @@
                 <include>org.apache.ratis.datastream.**</include>
                 <include>org.apache.ratis.server.**</include>
               </includes>
+              <excludedGroups>${flaky-test-groups}</excludedGroups>
             </configuration>
           </plugin>
         </plugins>
@@ -1153,6 +1156,21 @@
                 <exclude>org.apache.ratis.grpc.**</exclude>
                 <exclude>org.apache.ratis.server.**</exclude>
               </excludes>
+              <excludedGroups>${flaky-test-groups}</excludedGroups>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>flaky-tests</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <groups>${flaky-test-groups}</groups>
             </configuration>
           </plugin>
         </plugins>

diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml
@@ -33,6 +33,8 @@
     <test.cache.data>${project.build.directory}/test-classes</test.cache.data>
     <test.build.classes>${project.build.directory}/test-classes</test.build.classes>
     <license.bundles.dependencies>true</license.bundles.dependencies>
+    <!-- no testable code in this module -->
+    <skipTests>true</skipTests>
   </properties>
 
   <build>

diff --git a/ratis-client/pom.xml b/ratis-client/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-client</artifactId>
   <name>Apache Ratis Client</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ratis</groupId>

diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.test.tag;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.Tag;
+
+/**
+ * Annotation to mark JUnit5 test classes or methods that exhibit intermittent
+ * issues.  These are run separately from the normal tests in CI.  In case of
+ * failure they may be repeated a few times.
+ * Usage: <code>@Flaky("RATIS-123")</code>
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+@Retention(RetentionPolicy.RUNTIME)
+@Tag("flaky")
+public @interface Flaky {
+  /**
+   * The issue(s) tracking the flaky test.
+   */
+  String[] value();
+}
diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.test.tag;
+
+/**
+ * Interface to mark JUnit4 test classes or methods that exhibit intermittent
+ * issues.  These are run separately from the normal tests in CI.  In case of
+ * failure they may be repeated a few times.
+ * Usage: <code>@Category(FlakyTest.class) @Flaky("RATIS-123")</code>
+ */
+public interface FlakyTest {
+  // category marker
+}
diff --git a/ratis-docs/pom.xml b/ratis-docs/pom.xml
@@ -26,4 +26,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <artifactId>ratis-docs</artifactId>
   <name>Apache Ratis Documentation</name>
   <packaging>jar</packaging>
+
+  <properties>
+    <!-- no testable code in this module -->
+    <skipTests>true</skipTests>
+  </properties>
+
 </project>
diff --git a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java
@@ -24,6 +24,7 @@
 import org.apache.ratis.protocol.RaftGroup;
 import org.apache.ratis.server.impl.GroupManagementBaseTest;
 import org.apache.ratis.server.impl.MiniRaftCluster;
+import org.apache.ratis.test.tag.Flaky;
 import org.apache.ratis.util.function.CheckedBiConsumer;
 import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -33,6 +34,7 @@
 import java.util.Collection;
 import java.util.concurrent.atomic.AtomicInteger;
 
+@Flaky("RATIS-2218")
 @Timeout(value = 300)
 public class TestMultiRaftGroup extends BaseTest {
   public static Collection<Object[]> data() {

diff --git a/ratis-experiments/pom.xml b/ratis-experiments/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-experiments</artifactId>
   <name>Apache Ratis Experiments</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ratis</groupId>

diff --git a/ratis-metrics-api/pom.xml b/ratis-metrics-api/pom.xml
@@ -24,6 +24,11 @@
   <artifactId>ratis-metrics-api</artifactId>
   <name>Apache Ratis Metrics API</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <artifactId>ratis-common</artifactId>

diff --git a/ratis-netty/pom.xml b/ratis-netty/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-netty</artifactId>
   <name>Apache Ratis Netty Support</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ratis</groupId>

diff --git a/ratis-proto/pom.xml b/ratis-proto/pom.xml
@@ -25,6 +25,8 @@
 
   <properties>
     <maven.javadoc.skip>true</maven.javadoc.skip>
+    <!-- no testable code in this module -->
+    <skipTests>true</skipTests>
     <spotbugs.skip>true</spotbugs.skip>
   </properties>
 

diff --git a/ratis-replicated-map/pom.xml b/ratis-replicated-map/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-replicated-map</artifactId>
   <name>Apache Ratis Replicated Map</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ratis</groupId>

diff --git a/ratis-server-api/pom.xml b/ratis-server-api/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-server-api</artifactId>
   <name>Apache Ratis Server API</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ratis</groupId>

diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java
@@ -37,6 +37,7 @@
 import org.apache.ratis.server.RaftServerConfigKeys;
 import org.apache.ratis.server.metrics.LeaderElectionMetrics;
 import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.test.tag.Flaky;
 import org.apache.ratis.thirdparty.com.codahale.metrics.Timer;
 import org.apache.ratis.util.CodeInjectionForTesting;
 import org.apache.ratis.util.JavaUtils;
@@ -729,6 +730,7 @@ void runTestLeaderLease(CLUSTER cluster, long leaseTimeoutMs) throws Exception {
     }
   }
 
+  @Flaky("RATIS-2108")
   @Test
   public void testLeaderLeaseDuringReconfiguration() throws Exception {
     // use a strict lease

diff --git a/ratis-shell/pom.xml b/ratis-shell/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-shell</artifactId>
   <name>Apache Ratis Shell</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <artifactId>ratis-client</artifactId>

diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
@@ -28,6 +28,7 @@
 import org.apache.ratis.server.raftlog.RaftLog;
 import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing;
 import org.apache.ratis.statemachine.StateMachine;
+import org.apache.ratis.test.tag.Flaky;
 import org.apache.ratis.util.JavaUtils;
 import org.apache.ratis.util.TimeDuration;
 import org.junit.jupiter.api.Assertions;
@@ -55,6 +56,7 @@ public void testWithLoad() {
     // skip testWithLoad() from parent, called from parameterized testWithLoad(boolean)
   }
 
+  @Flaky("RATIS-2253")
   @ParameterizedTest
   @ValueSource(booleans = {true, false})
   public void testWithLoad(boolean separateHeartbeat) throws Exception {

diff --git a/ratis-tools/pom.xml b/ratis-tools/pom.xml
@@ -23,6 +23,11 @@
   <artifactId>ratis-tools</artifactId>
   <name>Apache Ratis Tools</name>
 
+  <properties>
+    <!-- no tests in this module so far -->
+    <skipTests>true</skipTests>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ratis</groupId>
-Original file line number
+Diff line change
@@ Expand Up / @@ -83,6 +83,7 @@ jobs: @@
               - grpc
               - server
               - misc
+              - flaky
           fail-fast: false
         uses: ./.github/workflows/check.yaml
         with:
@@ Expand Down @@