fix: optimising python integration tests

GoogleCloudPlatform · Nov 4, 2024 · 8d49d30 · 8d49d30
1 parent 7f768ab
commit 8d49d30
Showing 1 changed file with 72 additions and 0 deletions.
diff --git a/python/.ci/Jenkinsfile b/python/.ci/Jenkinsfile
@@ -68,6 +68,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 1'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{ 
                 stage('Hive TO GCS') {
                     steps{
@@ -155,6 +164,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 2'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{
                 stage('JDBC To JDBC') {
                     steps{
@@ -238,6 +256,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 3'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{
                 stage('GCS TO BIGQUERY') {
                     steps{
@@ -325,6 +352,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 4'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{
                 stage('MONGO TO GCS') {
                     steps{
@@ -477,6 +513,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 5'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{
                 stage('JDBC TO GCS') {
                     steps{
@@ -568,6 +613,15 @@ pipeline {
             }
         }  
          stage('Parallel Execution 6'){
+             when {
+                 expression {
+                     // Check if there are any changes in Python files in the last commit
+                     return sh(
+                         script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                         returnStatus: true
+                     ) == 0
+                 }
+             }
             parallel{
                 stage('S3 TO BigQuery (avro)') {    
                     steps{
@@ -604,6 +658,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 7'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{
                 // stage('Hbase TO GCS (Manual)') {    
                 //     when {
@@ -726,6 +789,15 @@ pipeline {
             }
         }
         stage('Parallel Execution 8'){
+            when {
+                expression {
+                    // Check if there are any changes in Python files in the last commit
+                    return sh(
+                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
+                        returnStatus: true
+                    ) == 0
+                }
+            }
             parallel{
                 stage('Elasticsearch to GCS'){
                     steps{