Laod test suite

.github/workflows/load-test.yml

@@ -35,7 +35,7 @@ jobs:
         run: mvn clean compile package -DskipTests=true
       - name: execute py script
         run: |
-          python loadTest.py
+          python perf_test/perfTestRunner.py
           git config user.name nitish
           git config user.email [email protected]
           git add .

common/core/src/main/java/zingg/common/core/block/Canopy.java

@@ -16,6 +16,7 @@
 
 
 public class Canopy<R> implements Serializable {
+	private static final long serialVersionUID = -229533781044789499L;
 
 	public static final Log LOG = LogFactory.getLog(Canopy.class);
 

perf_test_input.py → perf_test/perfTestInput.py

@@ -5,15 +5,16 @@
 INCREMENTAL = "runIncrement"
 
 #load file config to test on
-config_65 = "examples/febrl120k/config.json"
-config_120k = "examples/febrl120k/config120k.json"
+febrl = "./examples/febrl120k/config.json"
+febrl_120k = "./examples/febrl120k/config120k.json"
+ncVoter_5m = "./examples/ncVoters5M/config.json"
 
 
 #bash script location
-ZINGG = "scripts/zingg.sh"
+ZINGG = "./scripts/zingg.sh"
 
 
 #add all the load to test
-load_configs = {"65_samples" : config_65}
+load_configs = {"65_samples" : febrl, "120k_samples" : febrl_120k, "5m_samples" : ncVoter_5m}
 #add all the phases on which testing is required
 phases = [FIND_TRAINING_DATA, MATCH]

loadTest.py → perf_test/perfTestRunner.py

@@ -1,34 +1,44 @@
 import subprocess
-from perf_test_input import phases, load_configs, ZINGG
+from perfTestInput import phases, load_configs, ZINGG
 import time
-from datetime import date
+from datetime import date, datetime
 from subprocess import PIPE
+import os
+
+#set working directory
+os.chdir(os.path.dirname("../"))
 
 ZINGG = ZINGG           
 #phases to run: ftd, match
 phases_to_test = phases
 
+now = datetime.now()
+current_time = now.strftime("%H:%M:%S")
+
 #load to test: 65, 120k, 5m
 load = load_configs
 
 start_time = time.time()
 
+reportFile = "./perf_test/perf_test_report/loadTestReport"
+
 def perf_test_small_all():
     return "small_test_running_all"
 
+propertyFile = "./config/zingg.conf"
 
 def run_phase(phase, conf):
     print("Running phase - " + phase)
-    return subprocess.call(ZINGG + " %s %s %s %s" % ("--phase", phase, "--conf", conf), shell=True)
+    return subprocess.call(ZINGG + " %s %s %s %s %s %s" % ("--phase", phase, "--conf", conf, "--properties-file", propertyFile), shell=True)
 
 def perf_test_small(phase):
     return "small_test_running"
 
 
 def write_on_start():
-    f = open("loadTestReport_" + str(start_time), "w+")
-    f.write("******************************** perf test report, " + str(date.today()) + " ********************************\n\n");
-    f.write("--------- Test bed details ---------\n")
+    f = open(reportFile, "w+")
+    f.write("******************************** perf test report, " + str(date.today()) + ", " + current_time + " ********************************\n\n");
+    f.write("------------ Test bed details ------------\n")
     f.write("Load samples: ")
     for load, config in load_configs.items():
         f.write(str(load) + " ")
@@ -37,27 +47,27 @@ def write_on_start():
     for phase in phases:
         f.write(phase + " ")
     f.write("\n")
-    f.write("------------------------------------\n\n")
+    f.write("------------------------------------------\n\n")
     f.close()
 
 def write_on_complete():
-    f = open("loadTestReport_" + str(start_time), "a+")
-    f.write("***********************************************************************************************\n\n\n\n\n\n")
+    f = open(reportFile, "a+")
+    f.write("********************************************************************************************************\n\n\n\n\n\n")
 
 
 
 
 def write_success_stats(phase_time, load):
-    f = open("loadTestReport_" + str(start_time), "a+")
+    f = open(reportFile, "a+")
     f.write("{:>50}".format("capturing for " + load) + "\n")
     f.write("PHASE {:>65}".format("TIME_TAKEN_IN_MINUTES") + "\n")
     for phase, time in phase_time.items():
-        f.write(success_message(phase, time/60) + "\n")
+        f.write(success_message(phase, round(time/60, 1)) + "\n")
     f.write("\n")
     f.close()
 
 def write_failure_stats(phase_error):
-    f = open("loadTestReport_" + str(start_time), "a+")
+    f = open(reportFile, "a+")
     for phase, error in phase_error.items():
         f.write(error_message(phase, error) + "\n\n")
     f.close()

perf_test/perf_test_report/loadTestReport

@@ -0,0 +1,18 @@
+******************************** perf test report, 2024-11-14, 18:09:04 ********************************
+
+------------ Test bed details ------------
+Load samples: 65_samples 
+Phases: findTrainingData match 
+------------------------------------------
+
+                          capturing for 65_samples
+PHASE                                             TIME_TAKEN_IN_MINUTES
+findTrainingData                                                    3.0
+match                                                               4.5
+
+********************************************************************************************************
+
+
+
+
+

scripts/zingg.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-#ZINGG_HOME=./assembly/target
 ZINGG_JARS=$ZINGG_HOME/zingg-0.4.1-SNAPSHOT.jar
 [email protected]
 LICENSE=zinggLicense.txt