From 7eed3f83853c8d3adf86fbf0741dc62bb78b6780 Mon Sep 17 00:00:00 2001 From: Geremia Taglialatela Date: Sun, 22 Sep 2024 09:27:42 +0200 Subject: [PATCH] Test difference among benchmark executions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to the order of run, we have 14% difference among executions for `(a:, **kwargs)` ### GitHub main tested before local ``` $ bundle exec ruby benchmarks.rb Will BENCHMARK_GEMS: memo_wise-github-main (1.10.0) memo_wise-local (1.10.0.dev) ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-darwin23] Warming up -------------------------------------- memo_wise-github-main (1.10.0): (a:, **kwargs) 125.193k i/100ms memo_wise-local (1.10.0.dev): (a:, **kwargs) 143.649k i/100ms Calculating ------------------------------------- memo_wise-github-main (1.10.0): (a:, **kwargs) 1.146M (±23.0%) i/s (872.35 ns/i) - 5.508M in 5.067507s memo_wise-local (1.10.0.dev): (a:, **kwargs) 1.309M (±18.0%) i/s (764.21 ns/i) - 6.464M in 5.169169s Comparison: memo_wise-local (1.10.0.dev): (a:, **kwargs): 1308539.2 i/s memo_wise-github-main (1.10.0): (a:, **kwargs): 1146322.9 i/s - same-ish: difference falls within error |Method arguments|`memo_wise-github-main` (1.10.0)| |--|--| |`(a:, **kwargs)`|1.14x| |Method arguments|| |--| |`(a:, **kwargs)`|| ``` ### local tested before GitHub main ``` $ bundle exec ruby benchmarks.rb Will BENCHMARK_GEMS: memo_wise-local (1.10.0.dev) memo_wise-github-main (1.10.0) ruby 3.3.5 (2024-09-03 revision ef084cc8f4) [x86_64-darwin23] Warming up -------------------------------------- memo_wise-local (1.10.0.dev): (a:, **kwargs) 130.824k i/100ms memo_wise-github-main (1.10.0): (a:, **kwargs) 150.842k i/100ms Calculating ------------------------------------- memo_wise-local (1.10.0.dev): (a:, **kwargs) 1.184M (±23.6%) i/s (844.93 ns/i) - 5.756M in 5.139460s memo_wise-github-main (1.10.0): (a:, **kwargs) 1.381M (±17.8%) i/s (723.94 ns/i) - 6.637M in 5.026159s Comparison: memo_wise-github-main (1.10.0): (a:, **kwargs): 1381332.2 i/s memo_wise-local (1.10.0.dev): (a:, **kwargs): 1183526.0 i/s - same-ish: difference falls within error |Method arguments|`memo_wise-github-main` (1.10.0)| |--|--| |`(a:, **kwargs)`|0.86x| |Method arguments|| |--| |`(a:, **kwargs)`|| ``` Ref: #349 --- benchmarks/Gemfile | 4 +- benchmarks/benchmarks.rb | 91 ---------------------------------------- lib/memo_wise/version.rb | 2 +- 3 files changed, 3 insertions(+), 94 deletions(-) diff --git a/benchmarks/Gemfile b/benchmarks/Gemfile index 5516c64..1328880 100644 --- a/benchmarks/Gemfile +++ b/benchmarks/Gemfile @@ -7,7 +7,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby ">= 2.7.7" gem "benchmark-ips", "2.14.0" -gem "gem_bench", "2.0.3" +gem "gem_bench", "2.0.5" # NOTE: Regarding `require: false` below # 1. GitHub version of MemoWise and the local source of MemoWise share a namespace @@ -28,4 +28,4 @@ else gem "memoizer", "1.0.3" end -gem "memo_wise", github: "panorama-ed/memo_wise", branch: "main", require: false +gem "memo_wise", github: "tagliala/memo_wise", branch: "chore/prefer-require-relative", require: false diff --git a/benchmarks/benchmarks.rb b/benchmarks/benchmarks.rb index 0880948..be3d3b4 100644 --- a/benchmarks/benchmarks.rb +++ b/benchmarks/benchmarks.rb @@ -26,36 +26,6 @@ memoization_method: :memo_wise, }, ), - GemBench::Jersey.new( - gem_name: "alt_memery", - trades: { - "Memery" => "AltMemery" - }, - metadata: { - activation_code: "include AltMemery", - memoization_method: :memoize, - }, - ), - GemBench::Jersey.new( - gem_name: "memoist3", - trades: { - "Memoist" => "MemoistThree" - }, - metadata: { - activation_code: "extend MemoistThree", - memoization_method: :memoize, - }, - ), - GemBench::Jersey.new( - gem_name: "memoist", - trades: { - "Memoist" => "MemoistOne" - }, - metadata: { - activation_code: "extend MemoistOne", - memoization_method: :memoize, - }, - ), ].each(&:doff_and_don) # Copies, re-namespaces, and requires each gem. # We've already installed the `memo_wise` version on the `main` branch from GitHub in the @@ -119,11 +89,6 @@ def benchmark_name end benchmarked_gems.push( BenchmarkGem.new(MemoWise, "prepend MemoWise", :memo_wise, LOCAL_BENCHMARK_NAME), - (BenchmarkGem.new(DDMemoize, "DDMemoize.activate(self)", :memoize, "ddmemoize") if defined?(DDMemoize)), - (BenchmarkGem.new(Dry::Core, "include Dry::Core::Memoizable", :memoize, "dry-core") if defined?(Dry::Core)), - (BenchmarkGem.new(Memery, "include Memery", :memoize, "memery") if defined?(Memery)), - (BenchmarkGem.new(Memoized, "include Memoized", :memoize, "memoized") if defined?(Memoized)), - (BenchmarkGem.new(Memoizer, "include Memoizer", :memoize, "memoizer") if defined?(Memoizer)) ) BENCHMARK_GEMS = benchmarked_gems.compact.shuffle @@ -189,55 +154,6 @@ def positional_splat_keyword_and_double_splat_args(a, *args, b:, **kwargs) # Each method within these benchmarks is initially run once to memoize the # result value, so our benchmark only tests memoized retrieval time. benchmark_lambdas = [ - lambda do |x, instance, benchmark_gem| - instance.no_args - - x.report("#{benchmark_gem.benchmark_name}: ()") do - instance.no_args - end - end, - lambda do |x, instance, benchmark_gem| - instance.one_positional_arg(1) - - x.report("#{benchmark_gem.benchmark_name}: (a)") do - instance.one_positional_arg(1) - end - end, - lambda do |x, instance, benchmark_gem| - instance.positional_args(1, 2) - - x.report("#{benchmark_gem.benchmark_name}: (a, b)") do - instance.positional_args(1, 2) - end - end, - lambda do |x, instance, benchmark_gem| - instance.one_keyword_arg(a: 1) - - x.report("#{benchmark_gem.benchmark_name}: (a:)") do - instance.one_keyword_arg(a: 1) - end - end, - lambda do |x, instance, benchmark_gem| - instance.keyword_args(a: 1, b: 2) - - x.report("#{benchmark_gem.benchmark_name}: (a:, b:)") do - instance.keyword_args(a: 1, b: 2) - end - end, - lambda do |x, instance, benchmark_gem| - instance.positional_and_keyword_args(1, b: 2) - - x.report("#{benchmark_gem.benchmark_name}: (a, b:)") do - instance.positional_and_keyword_args(1, b: 2) - end - end, - lambda do |x, instance, benchmark_gem| - instance.positional_and_splat_args(1, 2) - - x.report("#{benchmark_gem.benchmark_name}: (a, *args)") do - instance.positional_and_splat_args(1, 2) - end - end, lambda do |x, instance, benchmark_gem| instance.keyword_and_double_splat_args(a: 1, b: 2) @@ -245,13 +161,6 @@ def positional_splat_keyword_and_double_splat_args(a, *args, b:, **kwargs) instance.keyword_and_double_splat_args(a: 1, b: 2) end end, - lambda do |x, instance, benchmark_gem| - instance.positional_splat_keyword_and_double_splat_args(1, 2, b: 3, a: 4) - - x.report("#{benchmark_gem.benchmark_name}: (a, *args, b:, **kwargs)") do - instance.positional_splat_keyword_and_double_splat_args(1, 2, b: 3, a: 4) - end - end ] # We benchmark different cases separately, to ensure that slow performance in diff --git a/lib/memo_wise/version.rb b/lib/memo_wise/version.rb index 428f25a..a4fb14d 100644 --- a/lib/memo_wise/version.rb +++ b/lib/memo_wise/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module MemoWise - VERSION = "1.10.0" + VERSION = "1.10.0.dev" end