disable runner_v2 tests on classic streaming templates

Signed-off-by: Jeffrey Kinard <[email protected]>

.github/workflows/uw-tests.yml

@@ -37,9 +37,10 @@ jobs:
         id: variables
         run: |
           echo "unifiedWorkerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT
-          echo "testBranchName=release_${CANDIDATE_NAME}" >> $GITHUB_OUTPUT
+          echo "testBranchName=uw_release_${CANDIDATE_NAME}" >> $GITHUB_OUTPUT
           echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCloudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT
         env:
+          CANDIDATE_NAME: ${{ inputs.candidateName }}
           HARNESS_IMAGE: ${{ inputs.unifiedWorkerHarnessContainerImage }}
       - name: Checkout code
         uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0

cicd/cmd/run-it-tests/main.go

@@ -55,7 +55,7 @@ func main() {
 		mvnFlags.SkipCheckstyle(),
 		mvnFlags.SkipJib(),
 		mvnFlags.SkipShade(),
-		mvnFlags.RunIntegrationTests(),
+		mvnFlags.RunIntegrationTests(flags.UnifiedWorkerHarnessContainerImage() != ""),
 		mvnFlags.ThreadCount(4),
 		mvnFlags.IntegrationTestParallelism(3),
 		mvnFlags.StaticBigtableInstance("teleport"),

cicd/internal/workflows/maven-workflows.go

@@ -49,7 +49,7 @@ type MavenFlags interface {
 	SkipSpotlessCheck() string
 	SkipIntegrationTests() string
 	FailAtTheEnd() string
-	RunIntegrationTests() string
+	RunIntegrationTests(bool) string
 	RunIntegrationSmokeTests() string
 	RunLoadTests() string
 	RunLoadTestObserver() string
@@ -107,7 +107,10 @@ func (*mvnFlags) FailAtTheEnd() string {
 	return "-fae"
 }
 
-func (*mvnFlags) RunIntegrationTests() string {
+func (*mvnFlags) RunIntegrationTests(bool skipRunnerV2) string {
+	if skipRunnerV2 {
+		return "-PtemplatesIntegrationRunnerV2Tests"
+	}
 	return "-PtemplatesIntegrationTests"
 }
 

metadata/pom.xml

@@ -93,6 +93,15 @@
         <failIfNoTests>false</failIfNoTests>
       </properties>
     </profile>
+    <profile>
+      <id>templatesIntegrationRunnerV2Tests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <failIfNoTests>false</failIfNoTests>
+      </properties>
+    </profile>
     <!-- Skip tests for this common module when profile enabled -->
     <profile>
       <id>skipCommonTests</id>

metadata/src/main/java/com/google/cloud/teleport/metadata/SkipRunnerV2Test.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.cloud.teleport.metadata;
+
+/** Annotation that marks tests that can not run using Runner V2 on DataflowRunner. */
+public interface SkipRunnerV2Test {}

pom.xml

@@ -655,6 +655,63 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>templatesIntegrationRunnerV2Tests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <!-- Skip coverage checks, unit tests are skipped -->
+        <jacoco.skip>true</jacoco.skip>
+        <!-- Some modules may yield no integration tests -->
+        <failIfNoTests>false</failIfNoTests>
+        <!-- Parallelism settings. Default is 2, set to consider methods -->
+        <itParallelismType>classesAndMethods</itParallelismType>
+        <itParallelism>2</itParallelism>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${surefire.version}</version>
+            <configuration combine.self="override">
+              <systemProperties>
+                <property>
+                  <name>beamPythonVersion</name>
+                  <value>${beam-python.version}</value>
+                </property>
+                <property>
+                  <name>beamJavaVersion</name>
+                  <value>${beam.version}</value>
+                </property>
+                <property>
+                  <name>beamMavenRepo</name>
+                  <value>${beam-maven-repo}</value>
+                </property>
+              </systemProperties>
+              <includes>
+                <include>**/*IT.java</include>
+              </includes>
+              <excludes>
+                <exclude>**/KafkaToKafkaIT.java</exclude>
+              </excludes>
+              <excludedGroups>
+                ${direct-runner.tests}
+                com.google.cloud.teleport.metadata.SkipRunnerV2Test
+              </excludedGroups>
+              <groups>
+                ${integration.tests}
+              </groups>
+              <reuseForks>true</reuseForks>
+              <parallel>${itParallelismType}</parallel>
+              <threadCount>${itParallelism}</threadCount>
+              <trimStackTrace>false</trimStackTrace>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>pluginOutputDir</id>
       <activation>

v1/src/test/java/com/google/cloud/teleport/templates/DLPTextToBigQueryStreamingIT.java

@@ -20,6 +20,7 @@
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult;
 
 import com.google.cloud.bigquery.TableId;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.privacy.dlp.v2.CharacterMaskConfig;
 import com.google.privacy.dlp.v2.CryptoHashConfig;
@@ -61,7 +62,7 @@
 import org.slf4j.LoggerFactory;
 
 /** Integration test for {@link DLPTextToBigQueryStreaming} (Stream_DLP_GCS_Text_to_BigQuery). */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(DLPTextToBigQueryStreaming.class)
 @RunWith(JUnit4.class)
 public class DLPTextToBigQueryStreamingIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/PubSubToAvroIT.java

@@ -21,6 +21,7 @@
 
 import com.google.cloud.teleport.avro.AvroPubsubMessageRecord;
 import com.google.cloud.teleport.metadata.SkipDirectRunnerTest;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -55,7 +56,7 @@
 
 /** Integration test for {@link PubsubToAvro} PubSub to Avro. */
 // SkipDirectRunnerTest: PubsubIO doesn't trigger panes on the DirectRunner.
-@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class})
+@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(value = PubsubToAvro.class, template = "Cloud_PubSub_to_Avro")
 @RunWith(JUnit4.class)
 public class PubSubToAvroIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/PubSubToBigQueryIT.java

@@ -26,6 +26,7 @@
 import com.google.cloud.bigquery.StandardSQLTypeName;
 import com.google.cloud.bigquery.TableId;
 import com.google.cloud.bigquery.TableResult;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -58,7 +59,7 @@
 import org.junit.runners.JUnit4;
 
 /** Integration test for {@link PubSubToBigQuery} classic template. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @RunWith(JUnit4.class)
 public final class PubSubToBigQueryIT extends TemplateTestBase {
 

v1/src/test/java/com/google/cloud/teleport/templates/PubSubToDatadogIT.java

@@ -26,6 +26,7 @@
 import com.google.cloud.teleport.it.datadog.conditions.DatadogLogEntriesCheck;
 import com.google.cloud.teleport.metadata.DirectRunnerTest;
 import com.google.cloud.teleport.metadata.SkipDirectRunnerTest;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -53,7 +54,7 @@
 import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
 
 /** Integration test for {@link PubSubToDatadog} classic template. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(PubSubToDatadog.class)
 @RunWith(JUnit4.class)
 public class PubSubToDatadogIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/PubSubToPubSubIT.java

@@ -19,6 +19,7 @@
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline;
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult;
 
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -44,7 +45,7 @@
 import org.junit.runners.JUnit4;
 
 /** Integration test for {@link PubsubToPubsub} (Cloud_PubSub_to_Cloud_PubSub). */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(PubsubToPubsub.class)
 @RunWith(JUnit4.class)
 public class PubSubToPubSubIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/PubSubToSplunkIT.java

@@ -24,6 +24,7 @@
 import com.google.cloud.kms.v1.CryptoKey;
 import com.google.cloud.teleport.metadata.DirectRunnerTest;
 import com.google.cloud.teleport.metadata.SkipDirectRunnerTest;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -59,7 +60,7 @@
  *
  * <p>TODO - Change CustomSplunkResourceManager back when Beam 2.55 is released
  */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(PubSubToSplunk.class)
 @RunWith(JUnit4.class)
 public class PubSubToSplunkIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/PubSubTopicToBigQueryIT.java

@@ -23,6 +23,7 @@
 import com.google.cloud.bigquery.Schema;
 import com.google.cloud.bigquery.StandardSQLTypeName;
 import com.google.cloud.bigquery.TableId;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -48,7 +49,7 @@
 import org.junit.runners.JUnit4;
 
 /** Integration test for {@link PubSubToBigQuery} PubSub Topic to Bigquery. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(value = PubSubToBigQuery.class, template = "PubSub_to_BigQuery")
 @RunWith(JUnit4.class)
 public final class PubSubTopicToBigQueryIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/PubsubToTextIT.java

@@ -20,6 +20,7 @@
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult;
 
 import com.google.cloud.teleport.metadata.SkipDirectRunnerTest;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -47,7 +48,7 @@
 
 /** Integration test for {@link PubsubToText} (Cloud_PubSub_to_GCS_Text). */
 // SkipDirectRunnerTest: PubsubIO doesn't trigger panes on the DirectRunner.
-@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class})
+@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(PubsubToText.class)
 @RunWith(JUnit4.class)
 public final class PubsubToTextIT extends TemplateTestBase {

v1/src/test/java/com/google/cloud/teleport/templates/TextToPubsubStreamIT.java

@@ -20,6 +20,7 @@
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline;
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult;
 
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.io.ByteStreams;
 import com.google.pubsub.v1.SubscriptionName;
@@ -55,7 +56,7 @@
 import org.slf4j.LoggerFactory;
 
 /** Integration test for {@link TextToPubsubStream} template. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(TextToPubsubStream.class)
 @RunWith(JUnit4.class)
 public class TextToPubsubStreamIT extends TemplateTestBase {

v2/googlecloud-to-googlecloud/src/test/java/com/google/cloud/teleport/v2/templates/DLPTextToBigQueryStreamingIT.java

@@ -20,6 +20,7 @@
 import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult;
 
 import com.google.cloud.bigquery.TableId;
+import com.google.cloud.teleport.metadata.SkipRunnerV2Test;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.privacy.dlp.v2.CharacterMaskConfig;
 import com.google.privacy.dlp.v2.CryptoHashConfig;
@@ -63,7 +64,7 @@
 /**
  * Integration test for {@link DLPTextToBigQueryStreaming} (Stream_DLP_GCS_Text_to_BigQuery_Flex).
  */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SkipRunnerV2Test.class})
 @TemplateIntegrationTest(DLPTextToBigQueryStreaming.class)
 @RunWith(JUnit4.class)
 public class DLPTextToBigQueryStreamingIT extends TemplateTestBase {