update tests

Picovoice · May 1, 2024 · 0caf1e8 · 0caf1e8
1 parent 9a571d3
commit 0caf1e8
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 21 deletions.
diff --git a/.github/workflows/python-perf.yml b/.github/workflows/python-perf.yml
@@ -45,11 +45,11 @@ jobs:
         os: [ubuntu-latest, windows-latest, macos-latest]
         include:
           - os: ubuntu-latest
-            proc_performance_threshold_sec: 1.0
+            proc_performance_threshold_rtf: 5.0
           - os: windows-latest
-            proc_performance_threshold_sec: 1.0
+            proc_performance_threshold_rtf: 3.0
           - os: macos-latest
-            proc_performance_threshold_sec: 1.0
+            proc_performance_threshold_rtf: 3.0
 
     steps:
       - uses: actions/checkout@v3
@@ -70,7 +70,7 @@ jobs:
           python3 test_orca_perf.py
           --access-key ${{secrets.PV_VALID_ACCESS_KEY}}
           --num-test-iterations 10
-          --proc-performance-threshold-sec ${{matrix.proc_performance_threshold_sec}}
+          --proc-performance-threshold-rtf ${{matrix.proc_performance_threshold_rtf}}
 
   perf-self-hosted:
     runs-on: ${{ matrix.machine }}
@@ -81,17 +81,17 @@ jobs:
         machine: [rpi3-32, rpi3-64, rpi4-32, rpi4-64, rpi5-64, jetson]
         include:
           - machine: rpi3-32
-            proc_performance_threshold_sec: 4.0
+            proc_performance_threshold_rtf: 1.0
           - machine: rpi3-64
-            proc_performance_threshold_sec: 2.0
+            proc_performance_threshold_rtf: 1.0
           - machine: rpi4-32
-            proc_performance_threshold_sec: 2.0
+            proc_performance_threshold_rtf: 2.0
           - machine: rpi4-64
-            proc_performance_threshold_sec: 2.0
+            proc_performance_threshold_rtf: 2.0
           - machine: rpi5-64
-            proc_performance_threshold_sec: 1.0
+            proc_performance_threshold_rtf: 2.0
           - machine: jetson
-            proc_performance_threshold_sec: 2.0
+            proc_performance_threshold_rtf: 2.0
 
     steps:
       - uses: actions/checkout@v3
@@ -108,7 +108,7 @@ jobs:
           python3 test_orca_perf.py
           --access-key ${{secrets.PV_VALID_ACCESS_KEY}}
           --num-test-iterations 10
-          --proc-performance-threshold-sec ${{matrix.proc_performance_threshold_sec}}
+          --proc-performance-threshold-rtf ${{matrix.proc_performance_threshold_rtf}}
 
       - name: Machine state after
         working-directory: resources/.scripts

diff --git a/binding/python/test_orca.py b/binding/python/test_orca.py
@@ -48,7 +48,7 @@ def _test_audio(self, pcm: Sequence[int], ground_truth: Sequence[int]) -> None:
         pcm = pcm[:len(ground_truth)]  # compensate for discrepancies due to wav header
         self.assertEqual(len(pcm), len(ground_truth))
         for i in range(len(pcm)):
-            self.assertAlmostEqual(pcm[i], ground_truth[i], delta=100)
+            self.assertAlmostEqual(pcm[i], ground_truth[i], delta=500)
 
     def _test_equal_timestamp(self, timestamp: float, timestamp_truth: float) -> None:
         self.assertAlmostEqual(timestamp, timestamp_truth, places=2)

diff --git a/binding/python/test_orca_perf.py b/binding/python/test_orca_perf.py
@@ -25,7 +25,7 @@
 class OrcaPerformanceTestCase(unittest.TestCase):
     access_key: str
     num_test_iterations: int
-    proc_performance_threshold_sec: float
+    proc_performance_threshold_rtf: float
 
     def test_performance_proc(self) -> None:
         for model_path in get_model_paths():
@@ -34,29 +34,31 @@ def test_performance_proc(self) -> None:
                 library_path=default_library_path('../..'),
                 model_path=model_path)
 
-            perf_results = list()
+            num_audio_seconds = 0
+            num_proc_seconds = 0
             for i in range(self.num_test_iterations):
                 start = perf_counter()
-                _ = orca.synthesize(test_data.text)
+                pcm, _ = orca.synthesize(test_data.text)
                 if i > 0:
-                    perf_results.append(perf_counter() - start)
+                    num_audio_seconds += len(pcm) / orca.sample_rate
+                    num_proc_seconds += perf_counter() - start
 
             orca.delete()
 
-            avg_perf = sum(perf_results) / self.num_test_iterations
-            print("Average proc performance [model=%s]: %s seconds" % (os.path.basename(model_path), avg_perf))
-            self.assertLess(avg_perf, self.proc_performance_threshold_sec)
+            real_time_factor = num_audio_seconds / num_proc_seconds
+            print("Average proc performance[model=%s]: RTF = %s " % (os.path.basename(model_path), real_time_factor))
+            self.assertGreater(real_time_factor, self.proc_performance_threshold_rtf)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--access-key', required=True)
     parser.add_argument('--num-test-iterations', type=int, required=True)
-    parser.add_argument('--proc-performance-threshold-sec', type=float, required=True)
+    parser.add_argument('--proc-performance-threshold-rtf', type=float, required=True)
     args = parser.parse_args()
 
     OrcaPerformanceTestCase.access_key = args.access_key
     OrcaPerformanceTestCase.num_test_iterations = args.num_test_iterations
-    OrcaPerformanceTestCase.proc_performance_threshold_sec = args.proc_performance_threshold_sec
+    OrcaPerformanceTestCase.proc_performance_threshold_rtf = args.proc_performance_threshold_rtf
 
     unittest.main(argv=sys.argv[:1])