Fix for REST bugs and certain metrics in bench (#32)

small fixes for REST bugs and certain metrics in bench
fsspec · Apr 8, 2024 · 4c6be7b · 4c6be7b
1 parent d907c10
commit 4c6be7b
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 36 deletions.
diff --git a/alluxio-py-bench.py b/alluxio-py-bench.py
@@ -55,7 +55,7 @@ def init_main_parser():
     return parser
 
 
-def main(main_args, remaining_args, test_suite=AbstractBench):
+def main(main_args, test_suite=AbstractBench):
     if not test_suite:
         print("No test suite specified, bail.")
         return
@@ -67,23 +67,27 @@ def main(main_args, remaining_args, test_suite=AbstractBench):
         if processid <= 0:
             i_am_child = True
             print(f"Child Process:{i}")
+            before_time = time.time()
             while time.time() - start_time < main_args.runtime:
-                before_time = time.time()
                 result_metrics = test_suite.execute()
-                duration = time.time() - before_time
-                if result_metrics.get(Metrics.TOTAL_OPS):
-                    print(
-                        f"Benchmark against {remaining_args.op}: total ops: {result_metrics.get(Metrics.TOTAL_OPS)}, ops/second: {result_metrics[Metrics.TOTAL_OPS] / duration}"
-                    )
-                if result_metrics.get(Metrics.TOTAL_BYTES):
-                    print(
-                        f"Benchmark against {remaining_args.op}: total bytes: {result_metrics.get(Metrics.TOTAL_BYTES)}, bytes/second: {result_metrics[Metrics.TOTAL_BYTES] / duration}"
-                    )
+            duration = time.time() - before_time
+            if result_metrics.get(Metrics.TOTAL_OPS):
+                print(
+                    f"Benchmark against {test_suite.args.op}: "
+                    f"total ops: {result_metrics.get(Metrics.TOTAL_OPS)}, "
+                    f"ops/second: {result_metrics.get(Metrics.TOTAL_OPS) / duration}"
+                )
+            if result_metrics.get(Metrics.TOTAL_BYTES):
+                print(
+                    f"Benchmark against {test_suite.args.op}: "
+                    f"total bytes: {result_metrics.get(Metrics.TOTAL_BYTES)}, "
+                    f"bytes/second: {result_metrics.get(Metrics.TOTAL_BYTES) / duration}"
+                )
 
-                if not result_metrics:
-                    print(
-                        f"Benchmark against {remaining_args.op}: iteration: {remaining_args.iteration} total time: {duration} seconds"
-                    )
+            if not result_metrics:
+                print(
+                    f"Benchmark against {test_suite.args.op}: iteration: {test_suite.args.iteration} total time: {duration} seconds"
+                )
             print(f"Child Process:{i} exit")
             break
         else:
@@ -112,4 +116,4 @@ def main(main_args, remaining_args, test_suite=AbstractBench):
     elif main_args.testsuite == TestSuite.RAY.name:
         suite_parser = RayBench.RayArgumentParser(main_parser)
         testsuite = RayBench.RayBench(suite_parser.parse_args())
-    main(main_args, remaining_args, testsuite)
+    main(main_args, testsuite)
diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py
@@ -535,7 +535,8 @@ def read_range(self, file_path, offset, length):
             )
         except Exception as e:
             raise Exception(
-                f"Error when reading file {file_path}: error {e}"
+                f"Error when reading file {file_path}: error {e}: "
+                f"worker_host{worker_host}, worker_http_port:{worker_http_port}"
             ) from e
 
     def write_page(self, file_path, page_index, page_bytes):

diff --git a/tests/benchmark/AlluxioFSSpecBench.py b/tests/benchmark/AlluxioFSSpecBench.py
@@ -1,5 +1,4 @@
 import random
-import time
 from enum import Enum
 
 from alluxiofs import AlluxioFileSystem
@@ -60,7 +59,6 @@ def init(self):
         self.metrics = Metrics()
 
     def execute(self) -> Metrics:
-        start_time = time.time()
         for i in range(self.args.iteration):
             if self.args.op == Op.ls.value:
                 result_metrics = self.bench_ls()
@@ -78,21 +76,6 @@ def execute(self) -> Metrics:
                 )
             for _, (k, v) in enumerate(result_metrics.items()):
                 self.metrics.update(k, v)
-        duration = time.time() - start_time
-
-        if self.metrics.get(Metrics.TOTAL_OPS):
-            print(
-                f"Benchmark against {self.args.op}: total ops: {self.metrics.get(Metrics.TOTAL_OPS)}, ops/second: {result_metrics[Metrics.TOTAL_OPS] / duration}"
-            )
-        if self.metrics.get(Metrics.TOTAL_BYTES):
-            print(
-                f"Benchmark against {self.args.op}: total bytes: {self.metrics.get(Metrics.TOTAL_BYTES)}, bytes/second: {result_metrics[Metrics.TOTAL_BYTES] / duration}"
-            )
-
-        if not result_metrics:
-            print(
-                f"Benchmark against {self.args.op}: iteration: {self.args.iteration} total time: {duration} seconds"
-            )
         return self.metrics
 
     def traverse(self, path, directories, files):

diff --git a/tests/benchmark/AlluxioRESTBench.py b/tests/benchmark/AlluxioRESTBench.py
@@ -136,7 +136,7 @@ def testGetPage(self):
         try:
             response = self.session.get(
                 FULL_PAGE_URL_FORMAT.format(
-                    worker_host=self.worker,
+                    worker_host=self.worker_host,
                     http_port=ALLUXIO_WORKER_HTTP_SERVER_PORT_DEFAULT_VALUE,
                     path_id=self.args.fileid,
                     page_index=page_idx,
@@ -149,7 +149,7 @@ def testGetPage(self):
             self.metrics.update(Metrics.TOTAL_BYTES, content_len)
         except Exception as e:
             raise Exception(
-                f"Error ListFiles, path:{self.path}: error {e}"
+                f"Error GetPage, path:{self.path}: error {e}"
             ) from e
 
     def testListFiles(self):