Setup benchmark suite

Signed-off-by: Tim Paine <[email protected]>

Author: timkpaine

.gitignore

@@ -104,6 +104,9 @@ csp/lib/
 *.so
 *.tsbuildinfo
 
+# Benchmarks
+.asv
+
 # Jupyter / Editors
 .ipynb_checkpoints
 .autoversion

Makefile

@@ -118,6 +118,14 @@ dockerps:  ## spin up docker compose services for adapter testing
 dockerdown:  ## spin up docker compose services for adapter testing
 	$(DOCKER) compose -f ci/$(ADAPTER)/docker-compose.yml down
 
+##############
+# BENCHMARKS #
+##############
+.PHONY: benchmark benchmarks
+benchmark:  ## run benchmarks
+	python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose
+
+
 ###########
 # VERSION #
 ###########

conda/dev-environment-unix.yml

@@ -3,6 +3,7 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
+ - asv
  - bison
  - brotli
  - build

conda/dev-environment-win.yml

@@ -3,6 +3,7 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
+ - asv
  - brotli
  - build
  - bump2version>=1

csp/benchmarks/__init__.py

@@ -0,0 +1,34 @@
+// https://asv.readthedocs.io/en/v0.6.3/asv.conf.json.html
+{
+    "version": 1,
+    "project": "csp",
+    "project_url": "https://github.com/Point72/csp",
+    "repo": "../..",
+    "branches": ["main"],
+    "dvcs": "git",
+
+    "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"],
+    "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
+    "build_command": [
+        "python -m pip install build",
+        "python -m build",
+        "python -mpip wheel -w {build_cache_dir} {build_dir}"
+    ],
+    "environment_type": "virtualenv",
+    "install_timeout": 600,
+    "show_commit_url": "http://github.com/point72/csp/commit/",
+
+    "pythons": ["3.11"],
+
+    // "environment_type": "mamba",
+    // "conda_channels": ["conda-forge"],
+    // "conda_environment_file": "conda/dev-environment-unix.yml",
+
+    "benchmark_dir": "../../csp/benchmarks",
+    "env_dir": "../../.asv/env",
+    "results_dir": "../../.asv/results",
+    "html_dir": "../../.asv/html",
+
+    "hash_length": 8,
+    "build_cache_size": 2
+}

csp/benchmarks/asv.conf.jsonc

@@ -0,0 +1,57 @@
+import numpy as np
+import time
+from datetime import datetime, timedelta
+
+import csp
+
+
+class StatsBenchmarkSuite:
+    def setup(self):
+        self.st = datetime(2020, 1, 1)
+        self.N = 10_000
+        self.ARRAY_SIZE = 100
+        self.TEST_TIMES = [self.st + timedelta(seconds=i) for i in range(self.N)]
+        self.RANDOM_VALUES = [np.random.normal(size=(self.ARRAY_SIZE,)) for i in range(self.N)]  # 100 element np array
+        self.DATA = list(zip(self.TEST_TIMES, self.RANDOM_VALUES))
+        self.INTERVAL = 1000
+        self.NUM_SAMPLES = 100
+
+    def time_stats_qtl(self):
+        def g_qtl():
+            data = csp.curve(typ=np.ndarray, data=self.DATA)
+            median = csp.stats.median(data, interval=self.INTERVAL)
+            csp.add_graph_output("final_median", median, tick_count=1)
+
+        qtl_times = []
+
+        for _ in range(self.NUM_SAMPLES):
+            start = time.time()
+            csp.run(g_qtl, starttime=self.st, endtime=timedelta(seconds=self.N))
+            post_qtl = time.time()
+            qtl_times.append(post_qtl - start)
+
+        avg_med = sum(qtl_times) / self.NUM_SAMPLES
+        print(
+            f"Average time in {self.NUM_SAMPLES} tests for median with {self.N=}, {self.ARRAY_SIZE=}, {self.INTERVAL=}: {round(avg_med, 2)} s"
+        )
+        return avg_med
+
+    def time_stats_rank(self):
+        def g_rank(self):
+            data = csp.curve(typ=np.ndarray, data=self.DATA)
+            rank = csp.stats.rank(data, interval=self.INTERVAL)
+            csp.add_graph_output("final_rank", rank, tick_count=1)
+
+        rank_times = []
+
+        for _ in range(self.NUM_SAMPLES):
+            start = time.time()
+            csp.run(g_rank, starttime=self.st, endtime=timedelta(seconds=self.stN))
+            post_rank = time.time()
+            rank_times.append(post_rank - start)
+
+        avg_rank = sum(rank_times) / self.NUM_SAMPLES
+        print(
+            f"Average time in {self.NUM_SAMPLES} tests for rank with {self.N=}, {self.ARRAY_SIZE=}, {self.INTERVAL=}: {round(avg_rank, 2)} s"
+        )
+        return avg_rank

csp/benchmarks/stats/__init__.py

@@ -83,6 +83,8 @@ develop = [
     "sqlalchemy",  # db
     "threadpoolctl",  # test_random
     "tornado",  # profiler, perspective, websocket
+    # benchmarks
+    "asv",
 ]
 showgraph = [
     "graphviz",