feat: benchmarks (#33)

* feat: benchmark macros * feat: biguint divmod benchmark * chore: makefile targets for running benchmarks * chore: script to prettify benchmark result into markdown table * ci: upload benchmark result to pr comment * chore: fix type in build deps definition * ci: fix run_benchmarks checkout * ci: fix run_benchmarks prettify * ci: allow write permission * ci: fixed comment message
MarcosNicolau · Jan 27, 2025 · d9f1790 · d9f1790
1 parent 1e3f23c
commit d9f1790
Show file tree

Hide file tree

Showing 10 changed files with 156 additions and 11 deletions.
diff --git a/.github/workflows/run_benchmarks.yaml b/.github/workflows/run_benchmarks.yaml
@@ -0,0 +1,28 @@
+name: "Benchmarks"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: ["*"]
+    paths:
+      - "libs/**"
+
+jobs:
+  run_benchmarks:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Compile"
+        run: make build
+      - name: "Run benchmarks"
+        run: |
+          make benchmark > benchmark_results.txt
+          make benchmark_prettify BENCHMARK_FILE=benchmark_results.txt > benchmark_table.md
+      - name: "Post benchmark results in pr comment"
+        uses: mshick/add-pr-comment@v2
+        with:
+          message-path: benchmark_table.md
diff --git a/Makefile b/Makefile
@@ -5,12 +5,14 @@ LIBS = utils primitive-types math hashes digital-signature # List of libraries i
 SRC_DIR = src
 INCLUDE_DIR = include
 TEST_DIR = tests
+BENCHMARKS_DIR = benchmarks
 
 # Output directories
 LIB_BUILD_DIR = $(BUILD_DIR)/lib
 INCLUDE_BUILD_DIR = $(BUILD_DIR)/include/$(PROJECT_NAME)
 OBJ_BUILD_DIR = $(BUILD_DIR)/obj
 TESTS_BUILD_DIR = $(BUILD_DIR)/tests
+BENCHMARKS_BUILD_DIR = $(BUILD_DIR)/benchmarks
 
 # Installation directories
 PREFIX = /usr/local
@@ -32,7 +34,10 @@ LDFLAGS = -shared
 help:
 	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-build: headers $(patsubst %, $(LIB_BUILD_DIR)/lib%.so, $(LIBS)) ## Build all libraries
+build:
+	@for lib in $(LIBS); do \
+		$(MAKE) build_$$lib > /dev/null 2>&1; \
+	done
 
 build_%: 
 	@$(MAKE) headers_$*
@@ -43,7 +48,7 @@ install: headers_install $(patsubst %, $(INSTALL_LIB_DIR)/libalmunecar_%.so, $(L
 # Build shared library for each lib into build directory
 $(LIB_BUILD_DIR)/lib%.so: $(OBJ_BUILD_DIR)/% | $(LIB_BUILD_DIR) $(OBJ_BUILD_DIR)
 	$(eval include libs/$*/deps.mk)
-	@$(CC) $(LDFLAGS) $(wildcard $(OBJ_BUILD_DIR)/$*/*.o) -L$(LIB_BUILD_DIR) $(patsubst %, -l%, $(DEPS)) -o $@
+	@$(CC) $(LDFLAGS) $(wildcard $(OBJ_BUILD_DIR)/$*/*.o) -L$(LIB_BUILD_DIR) $(patsubst %, -l%, $(BUILD_DEPS)) -o $@
 
 # Build objects of each library into build dir
 $(OBJ_BUILD_DIR)/%: 
@@ -55,16 +60,16 @@ $(OBJ_BUILD_DIR)/%:
 # Build shared library for each lib into INSTALL_LIB_DIR directory
 $(INSTALL_LIB_DIR)/libalmunecar_%.so: $(OBJ_BUILD_DIR)/%
 	$(eval include libs/$*/deps.mk)
-	@$(CC) $(LDFLAGS) $(wildcard $(OBJ_BUILD_DIR)/$*/*.o) -L$(INSTALL_LIB_DIR) $(patsubst %, -l%, $(DEPS)) -o $@ 
+	@$(CC) $(LDFLAGS) $(wildcard $(OBJ_BUILD_DIR)/$*/*.o) -L$(INSTALL_LIB_DIR) $(patsubst %, -l%, $(BUILD_DEPS)) -o $@ 
 
 # Create necessary directories
-$(LIB_BUILD_DIR) $(INCLUDE_BUILD_DIR) $(OBJ_BUILD_DIR) $(TESTS_BUILD_DIR):
+$(LIB_BUILD_DIR) $(INCLUDE_BUILD_DIR) $(OBJ_BUILD_DIR) $(TESTS_BUILD_DIR) $(BENCHMARKS_BUILD_DIR):
 	@mkdir -p $@
 
 # Copy headers to the include directory
 headers: $(INCLUDE_BUILD_DIR)
 	@for lib in $(LIBS); do \
-		$(MAKE) headers_$$lib; \
+		$(MAKE) headers_$$lib > /dev/null 2>&1; \
 	done
 
 headers_%: $(INCLUDE_BUILD_DIR)
@@ -96,13 +101,35 @@ test_%: build $(TESTS_BUILD_DIR)
 	@for test in libs/$*/$(TEST_DIR)/*.c; do \
 		mkdir -p $(TESTS_BUILD_DIR)/$*; \
 		$(eval include libs/$*/deps.mk) \
-		$(CC) $(CFLAGS) -I$(INCLUDE_BUILD_DIR) -o $(TESTS_BUILD_DIR)/$*/$$(basename $$test .c) $$test -L$(LIB_BUILD_DIR) $(patsubst %, -l%, $(DEPS)) -l$*; \
+		$(CC) $(CFLAGS) -I$(INCLUDE_BUILD_DIR) -o $(TESTS_BUILD_DIR)/$*/$$(basename $$test .c) $$test -L$(LIB_BUILD_DIR) $(patsubst %, -l%, $(TESTS_DEPS)) -l$*; \
 		FAIL_FAST=$(FAIL_FAST) LD_LIBRARY_PATH=$(LIB_BUILD_DIR) $(TESTS_BUILD_DIR)/$*/$$(basename $$test .c); \
 		if [ $$? -ne 0 ]; then \
 			exit 1; \
 		fi; \
 	done
 
+benchmark: build $(BENCHMARKS_BUILD_DIR) ## Run benchmarks for all libs. To run only the benchmarks of a specific lib run do benchmark_<LIB_NAME>, for example: make benchmark_primitive-types. 
+	@for lib in $(LIBS); do \
+		$(MAKE) benchmark_$$lib; 	\
+	done
+
+benchmark_%: build $(BENCHMARKS_BUILD_DIR) 
+	@if [ -d "libs/$*/$(BENCHMARKS_DIR)" ]; then \
+		for benchmark in libs/$*/$(BENCHMARKS_DIR)/*.c; do \
+			mkdir -p $(BENCHMARKS_BUILD_DIR)/$*; \
+			$(eval include libs/$*/deps.mk) \
+			$(CC) $(CFLAGS) -I$(INCLUDE_BUILD_DIR) -o $(BENCHMARKS_BUILD_DIR)/$*/$$(basename $$benchmark .c) $$benchmark -L$(LIB_BUILD_DIR) $(patsubst %, -l%, $(BENCHMARKS_DEPS)) -l$*; \
+			FAIL_FAST=$(FAIL_FAST) LD_LIBRARY_PATH=$(LIB_BUILD_DIR) $(BENCHMARKS_BUILD_DIR)/$*/$$(basename $$benchmark .c); \
+			if [ $$? -ne 0 ]; then \
+				exit 1; \
+			fi; \
+		done \
+	fi
+
+benchmark_prettify: ## Given a file with the output of a benchmark, it runs a prettify script to show it as a markdown table
+	@./scripts/prettify_benchmark.sh $(BENCHMARK_FILE)
+
+
 check_fmt: ## Checks formatting and outputs the diff
 	@./scripts/fmt.sh libs
 

diff --git a/libs/digital-signature/deps.mk b/libs/digital-signature/deps.mk
@@ -1 +1,3 @@
-DEPS := utils primitive-types math
+BUILD_DEPS := utils primitive-types math
+TESTS_DEPS := utils primitive-types math
+BENCHMARKS_DEPS := utils primitive-types math
diff --git a/libs/hashes/deps.mk b/libs/hashes/deps.mk
@@ -1 +1,3 @@
-DEPS := utils primitive-types
+BUILD_DEPS := utils primitive-types
+TESTS_DEPS := utils primitive-types
+BENCHMARKS_DEPS := utils primitive-types
diff --git a/libs/math/deps.mk b/libs/math/deps.mk
@@ -1 +1,3 @@
-DEPS := utils primitive-types
+BUILD_DEPS := utils primitive-types
+TESTS_DEPS := utils primitive-types
+BENCHMARKS_DEPS := utils primitive-types
diff --git a/libs/primitive-types/benchmarks/biguint.c b/libs/primitive-types/benchmarks/biguint.c
@@ -0,0 +1,17 @@
+#include <math/random.h>
+#include <primitive-types/biguint.h>
+#include <utils/benchmark.h>
+
+void benchmark_divmod() {
+    BigUint a = biguint_new(4);
+    BigUint b = biguint_new(4);
+    biguint_random(&a);
+    biguint_random(&b);
+    biguint_divmod(a, b, &a, &b);
+}
+
+int main() {
+    BEGIN_BENCHMARK();
+    benchmark(benchmark_divmod, 1000000);
+    END_BENCHMARK();
+}
diff --git a/libs/primitive-types/deps.mk b/libs/primitive-types/deps.mk
@@ -1 +1,3 @@
-DEPS := utils
+BUILD_DEPS := utils
+TESTS_DEPS := utils
+BENCHMARKS_DEPS := utils math
diff --git a/libs/utils/deps.mk b/libs/utils/deps.mk
@@ -1 +1,3 @@
-DEPS :=
+BUILD_DEPS := 
+TESTS_DEPS := 
+BENCHMARKS_DEPS := 
diff --git a/libs/utils/include/benchmark.h b/libs/utils/include/benchmark.h
@@ -0,0 +1,38 @@
+#ifndef TEST_H
+#define TEST_H
+#include <time.h>
+#include <utils/macros.h>
+
+#define benchmark(benchmark_fn, iterations, ...)                                                                       \
+    do {                                                                                                               \
+        printf("\n=============== %s (%d iterations) ===============\n", #benchmark_fn, iterations);                   \
+        int ANONYMOUS_VARIABLE(benchmark_fn) = 0;                                                                      \
+        double measures[iterations];                                                                                   \
+        for (; ANONYMOUS_VARIABLE(benchmark_fn) < iterations; ANONYMOUS_VARIABLE(benchmark_fn)++) {                    \
+            clock_t start_time = clock();                                                                              \
+            benchmark_fn(__VA_ARGS__);                                                                                 \
+            clock_t end_time = clock();                                                                                \
+            double diff = (double)(end_time - start_time) / CLOCKS_PER_SEC;                                            \
+            measures[ANONYMOUS_VARIABLE(benchmark_fn)] = diff;                                                         \
+        }                                                                                                              \
+        double sum = 0;                                                                                                \
+        double avg = 0;                                                                                                \
+        for (int ANONYMOUS_VARIABLE(benchmark_fn_i) = 0; ANONYMOUS_VARIABLE(benchmark_fn_i) < iterations;              \
+             ANONYMOUS_VARIABLE(benchmark_fn_i)++) {                                                                   \
+            sum += measures[ANONYMOUS_VARIABLE(benchmark_fn_i)];                                                       \
+        }                                                                                                              \
+        avg = sum / iterations;                                                                                        \
+        printf("Total execution took: %f seconds\n", sum);                                                             \
+        printf("Average execution took: %f seconds\n", avg);                                                           \
+        printf("=============================\n");                                                                     \
+    } while (0)
+
+#define BEGIN_BENCHMARK()                                                                                              \
+    printf("\n==============================\n");                                                                      \
+    printf("Benchmark suite %s at %s\n", __FILE__, __func__);
+
+#define END_BENCHMARK()                                                                                                \
+    printf("\nBenchmark suite %s at %s", __FILE__, __func__);                                                          \
+    printf("\n==============================\n");
+
+#endif
diff --git a/scripts/prettify_benchmark.sh b/scripts/prettify_benchmark.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+FILE=$1
+RESULTS=$(cat $FILE)
+
+# Initialize the Markdown table header
+echo "|Library| Benchmark Name | Iterations | Total Execution Time | Average Execution Time |"
+echo "|-------|----------------|------------|----------------------|------------------------|"
+
+current_library=""
+while IFS= read -r line; do
+    if [[ "$line" =~ Benchmark\ suite\ libs/([^/]+)/benchmarks ]]; then
+        current_library="${BASH_REMATCH[1]}"
+    elif [[ "$line" =~ ^\=\=\=\=\=\=\=\=+\ (.*)\ \(([0-9]+)\ iterations\)\ =\=\=\=\=\=\=\=\=+ ]]; then
+        benchmark_name="${BASH_REMATCH[1]}"
+        iterations="${BASH_REMATCH[2]}"
+        benchmark_name=$(echo "$benchmark_name" | sed 's/^benchmark_//')
+    elif [[ "$line" =~ Total\ execution\ took:\ ([0-9.]+)\ seconds ]]; then
+        total_time="${BASH_REMATCH[1]}"
+    elif [[ "$line" =~ Average\ execution\ took:\ ([0-9.]+)\ seconds ]]; then
+        avg_time="${BASH_REMATCH[1]}"
+        echo "| $current_library | $benchmark_name | $iterations | $total_time seconds | $avg_time seconds |"
+    fi
+done <<< "$RESULTS"
+