From 887e8a3b0cd45d88ae8ebbb1e32eb1907be6ffe4 Mon Sep 17 00:00:00 2001 From: Matthew Johnston Date: Mon, 27 Mar 2023 12:19:49 -0500 Subject: [PATCH] Add basic benchmarks to set a baseline (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I figure these benchmarks are good to have as a baseline. ``` Operating System: Linux CPU Information: AMD Ryzen 9 5900X 12-Core Processor Number of Available Cores: 24 Available memory: 125.72 GB Elixir 1.14.0 Erlang 25.0.4 Benchmark suite executing with the following configuration: warmup: 1 s time: 5 s memory time: 2 s reduction time: 2 s parallel: 1 inputs: none specified Estimated total run time: 1.17 min Benchmarking map_multiplcation ... Benchmarking nested_filter_comparison ... Benchmarking reduce_list_and_accumulate ... Benchmarking resolving_nested_value ... Benchmarking resolving_value ... Benchmarking simple_equals_equal ... Benchmarking simple_equals_not_equal ... Name ips average deviation median 99th % resolving_nested_value 1653.02 K 0.60 μs ±4165.86% 0.55 μs 0.78 μs resolving_value 1424.66 K 0.70 μs ±3379.18% 0.65 μs 0.84 μs simple_equals_not_equal 640.97 K 1.56 μs ±1441.87% 1.46 μs 1.89 μs simple_equals_equal 616.39 K 1.62 μs ±1537.39% 1.58 μs 1.84 μs map_multiplcation 348.05 K 2.87 μs ±986.31% 2.69 μs 3.67 μs nested_filter_comparison 256.74 K 3.89 μs ±531.92% 3.62 μs 4.75 μs reduce_list_and_accumulate 160.26 K 6.24 μs ±225.73% 4.59 μs 8.15 μs Comparison: resolving_nested_value 1653.02 K resolving_value 1424.66 K - 1.16x slower +0.0970 μs simple_equals_not_equal 640.97 K - 2.58x slower +0.96 μs simple_equals_equal 616.39 K - 2.68x slower +1.02 μs map_multiplcation 348.05 K - 4.75x slower +2.27 μs nested_filter_comparison 256.74 K - 6.44x slower +3.29 μs reduce_list_and_accumulate 160.26 K - 10.31x slower +5.63 μs Memory usage statistics: Name Memory usage resolving_nested_value 192 B resolving_value 128 B - 0.67x memory usage -64 B simple_equals_not_equal 784 B - 4.08x memory usage +592 B simple_equals_equal 784 B - 4.08x memory usage +592 B map_multiplcation 2320 B - 12.08x memory usage +2128 B nested_filter_comparison 4672 B - 24.33x memory usage +4480 B reduce_list_and_accumulate 2880 B - 15.00x memory usage +2688 B **All measurements for memory usage were the same** Reduction count statistics: Name Reduction count resolving_nested_value 0.0340 K resolving_value 4 K - 117.65x reduction count +3.97 K simple_equals_not_equal 4.05 K - 119.06x reduction count +4.01 K simple_equals_equal 4.05 K - 119.15x reduction count +4.02 K map_multiplcation 0.31 K - 9.03x reduction count +0.27 K nested_filter_comparison 0.42 K - 12.47x reduction count +0.39 K reduce_list_and_accumulate 20.05 K - 589.74x reduction count +20.02 K **All measurements for reduction count were the same** ``` --- benchmarks/basic.exs | 56 ++++++++++++++++++++++++++++++++++++++++++++ mix.exs | 1 + mix.lock | 3 +++ 3 files changed, 60 insertions(+) create mode 100644 benchmarks/basic.exs diff --git a/benchmarks/basic.exs b/benchmarks/basic.exs new file mode 100644 index 0000000..648aa96 --- /dev/null +++ b/benchmarks/basic.exs @@ -0,0 +1,56 @@ +defmodule Benchmarking do + def resolving_value do + JsonLogic.resolve(%{"var" => "key"}, %{"key" => "value"}) + end + + def resolving_nested_value do + JsonLogic.resolve(%{"var" => "nested.key"}, %{"nested" => %{"key" => "value"}}) + end + + def simple_equals_not_equal do + JsonLogic.resolve(%{"==" => [1, %{"var" => "key"}]}, %{"key" => 2}) + end + + def simple_equals_equal do + JsonLogic.resolve(%{"==" => [1, %{"var" => "key"}]}, %{"key" => 1}) + end + + def map_multiplcation do + JsonLogic.resolve(%{"map" => [[1,2,3,4,5], %{"*" => [%{"var" => ""}, 2]}]}) + end + + def nested_filter_comparison do + JsonLogic.resolve(%{"filter" => [[1,2,3,4,5], %{">" => [%{"var" => ""}, 2]}]}) + end + + def reduce_list_and_accumulate do + JsonLogic.resolve(%{ + "reduce" => [ + [1,2,3,4,5], + %{ + "+" => [ + %{"var" => "current"}, + %{"var" => "accumulator"} + ] + }, + 0 + ] + }) + end +end + +Benchee.run( + %{ + "map_multiplcation" => &Benchmarking.map_multiplcation/0, + "nested_filter_comparison" => &Benchmarking.nested_filter_comparison/0, + "reduce_list_and_accumulate" => &Benchmarking.reduce_list_and_accumulate/0, + "resolving_nested_value" => &Benchmarking.resolving_nested_value/0, + "resolving_value" => &Benchmarking.resolving_value/0, + "simple_equals_equal" => &Benchmarking.simple_equals_equal/0, + "simple_equals_not_equal" => &Benchmarking.simple_equals_not_equal/0, + }, + warmup: 1, + time: 5, + memory_time: 2, + reduction_time: 2 +) diff --git a/mix.exs b/mix.exs index 45aee32..b7bc165 100644 --- a/mix.exs +++ b/mix.exs @@ -45,6 +45,7 @@ defmodule JsonLogic.Mixfile do defp deps do [ {:decimal, ">= 0.0.0"}, + {:benchee, "~> 1.1", only: :dev}, {:credo, "~> 1.4", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.2.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.29", only: :dev, runtime: false} diff --git a/mix.lock b/mix.lock index d814255..e866008 100644 --- a/mix.lock +++ b/mix.lock @@ -1,7 +1,9 @@ %{ + "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"}, "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, "credo": {:hex, :credo, "1.6.7", "323f5734350fd23a456f2688b9430e7d517afb313fbd38671b8a4449798a7854", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "41e110bfb007f7eda7f897c10bf019ceab9a0b269ce79f015d54b0dcf4fc7dd3"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, + "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, "earmark_parser": {:hex, :earmark_parser, "1.4.30", "0b938aa5b9bafd455056440cdaa2a79197ca5e693830b4a982beada840513c5f", [:mix], [], "hexpm", "3b5385c2d36b0473d0b206927b841343d25adb14f95f0110062506b300cd5a1b"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, @@ -12,4 +14,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, + "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, }