From 21512b11fa4df28278d8c992c51494d75e4216a9 Mon Sep 17 00:00:00 2001 From: James McMullan Date: Fri, 15 Dec 2023 11:00:46 -0500 Subject: [PATCH] HPCC4J-561 copyfile test wait for spray completion - Modified WSFileIOClientTest.copyfile test to correctly monitor spray progress - Updated remote unit tests values.yaml to keep file spray service Signed-off-by: James McMullan James.McMullan@lexisnexis.com --- .github/workflows/httpsUnitTests.yml | 23 ++++++++- .../ws/client/WSFileIOClientTest.java | 47 +++++++++++-------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/.github/workflows/httpsUnitTests.yml b/.github/workflows/httpsUnitTests.yml index 215ff1bd3..5a1b43655 100644 --- a/.github/workflows/httpsUnitTests.yml +++ b/.github/workflows/httpsUnitTests.yml @@ -66,7 +66,28 @@ jobs: - name: Install HPCC Cluster run: | - echo -e "certificates:\n enabled: true\ndafilesrv:\n - name: rowservice\n disabled: false\n application: stream\n service:\n servicePort: 7600\n visibility: global" > values.yaml + cat < values.yaml + certificates: + enabled: true + dafilesrv: + - name: rowservice + disabled: false + application: stream + service: + servicePort: 7600 + visibility: global + - name: direct-access + disabled: true + application: directio + service: + servicePort: 7200 + visibility: local + - name: spray-service + application: spray + service: + servicePort: 7300 + visibility: cluster + EOF helm repo add hpcc https://hpcc-systems.github.io/helm-chart helm repo update helm install myhpcc hpcc/hpcc -f values.yaml diff --git a/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java b/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java index 158538608..e27714e24 100644 --- a/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java +++ b/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java @@ -21,7 +21,9 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®. import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; +import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.axis2.AxisFault; @@ -67,9 +69,6 @@ public class WSFileIOClientTest extends BaseRemoteTest @Test public void copyFile() throws Exception { - Assume.assumeFalse("Test not valid on containerized HPCC environment", client.isTargetHPCCContainerized()); - assumeTrue("Ignoring test 'copyFile' because HPCC-30117 is not fixed", HPCC_30117.equalsIgnoreCase("fixed")); - String lzfile=System.currentTimeMillis() + "_csvtest.csv"; String hpccfilename="temp::" + lzfile; client.createHPCCFile(lzfile, targetLZ, true); @@ -77,35 +76,43 @@ public void copyFile() throws Exception client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20); try { - ProgressResponseWrapper dfuspray=wsclient.getFileSprayClient().sprayVariable( + System.out.println("Starting file spray."); + ProgressResponseWrapper dfuspray = wsclient.getFileSprayClient().sprayVariable( new DelimitedDataOptions(), wsclient.getFileSprayClient().fetchLocalDropZones().get(0), lzfile,"~" + hpccfilename,"",thorClusterFileGroup,true, HPCCFileSprayClient.SprayVariableFormat.DFUff_csv, null, null, null, null, null, null, null); - Thread.sleep(1000); - int wait=60; if (dfuspray.getExceptions() != null - && dfuspray.getExceptions().getException() != null - && dfuspray.getExceptions().getException().size()>0) + && dfuspray.getExceptions().getException() != null + && dfuspray.getExceptions().getException().size()>0) { fail(dfuspray.getExceptions().getException().get(0).getMessage()); } - if (dfuspray.getSecsLeft()>0) + + List whiteListedStates = Arrays.asList( "queued", "started", "unknown", "finished", "monitoring"); + int waitCount = 0; + int MAX_WAIT_COUNT = 60; + + ProgressResponseWrapper dfuProgress = null; + do { - System.out.println("Still spraying, waiting 1 sec..."); - for (int i=wait;i>0;i--) + dfuProgress = wsclient.getFileSprayClient().getDfuProgress(dfuspray.getWuid()); + boolean stateIsWhiteListed = whiteListedStates.contains(dfuProgress.getState()); + if (!stateIsWhiteListed) { - if (dfuspray.getSecsLeft()==0) - { - i=0; - } - else - { - Thread.sleep(1000); - } + fail("File spray failed: Summary: " + dfuProgress.getSummaryMessage() + " Exceptions: " + dfuProgress.getExceptions()); } - } + + if (dfuProgress.getPercentDone() < 100) + { + Thread.sleep(1000); + System.out.println("File spray percent complete: " + dfuProgress.getPercentDone() + "% Sleeping for 1sec to wait for spray."); + waitCount++; + } + } while (waitCount < 60 && dfuProgress.getPercentDone() < 100); + + assumeTrue("File spray did not complete within: " + MAX_WAIT_COUNT + "s. Failing test.", waitCount < MAX_WAIT_COUNT); System.out.println("Test file successfully sprayed to " + "~" + hpccfilename + ", attempting copy to " + hpccfilename + "_2"); wsclient.getFileSprayClient().copyFile(hpccfilename,hpccfilename + "_2",true);