From 936ab3f0a37e066c0e6e7f025b6784d663271a55 Mon Sep 17 00:00:00 2001 From: Peter Boling Date: Tue, 17 Sep 2024 01:43:49 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=9D=20New=20benchmarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruby 3.3.5: - alt_memery - memoist3 --- CHANGELOG.md | 4 +-- README.md | 54 ++++++++++++++++++++-------------------- benchmarks/Gemfile | 11 +++++--- benchmarks/benchmarks.rb | 42 ++++++++++++++++++------------- 4 files changed, 62 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0869a4..2d9b972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ _No breaking changes!_ **Project enhancements:** - Updated official test coverage to support Ruby 3.3 [[#335](https://github.com/panorama-ed/memo_wise/pull/335)] +- Added `alt_memery` and `memoist3` to benchmarks [#339](https://github.com/panorama-ed/memo_wise/pull/339) + - Updated benchmark results in `README.md` to Ruby 3.3.5 ## [v1.9.0](https://github.com/panorama-ed/memo_wise/compare/v1.8.0...v1.9.0) @@ -29,8 +31,6 @@ _Breaking changes:_ - Fixed `bundle exec yard server --reload` and related documentation [[#333]](https://github.com/panorama-ed/memo_wise/pull/333) - Fixed Codecov rate limiting errors affecting pull requests by upgrading `codecov/codecov-action` and using a Codecov token [[#317]](https://github.com/panorama-ed/memo_wise/pull/317) -- Added alt_memery and memoist3 to benchmarks [#339](https://github.com/panorama-ed/memo_wise/pull/339) - - Updated benchmark results in `README.md` to Ruby 3.3.2 ## [v1.8.0](https://github.com/panorama-ed/memo_wise/compare/v1.7.0...v1.8.0) - 2023-10-25 diff --git a/README.md b/README.md index 153fc20..9dcdbcc 100644 --- a/README.md +++ b/README.md @@ -114,37 +114,37 @@ For more usage details, see our detailed [documentation](#documentation). Benchmarks are run in GitHub Actions, and the tables below are updated with every code change. **Values >1.00x represent how much _slower_ each gem’s memoized value retrieval is than the latest commit of `MemoWise`**, according to [`benchmark-ips`](https://github.com/evanphx/benchmark-ips) (2.11.0). -Results using Ruby 3.3.2: - -|Method arguments|`Dry::Core` (1.0.1)|`Memery` (alt_memery|`Memoist` (memoist3| -|--|--|--|--| -|`()` (none)|0.44x|10.91x|3.73x| -|`(a)`|0.87x|6.65x|14.31x| -|`(a, b)`|0.69x|4.80x|9.86x| -|`(a:)`|0.98x|10.12x|19.52x| -|`(a:, b:)`|0.71x|8.49x|17.15x| -|`(a, b:)`|0.70x|8.03x|13.98x| -|`(a, *args)`|0.80x|2.09x|3.62x| -|`(a:, **kwargs)`|0.63x|2.46x|6.36x| -|`(a, *args, b:, **kwargs)`|0.68x|1.69x|4.03x| +Results using Ruby 3.3.5: + +| Method arguments | `alt_memery` (2.1.0) | `dry-core` \* (1.0.1) | `memery` (1.6.0) | `memoist3` (1.0.0) | +|----------------------------|----------------------|-----------------------|------------------|--------------------| +| `()` (none) | 11.86x | 0.45x | 5.23x | 3.72x | +| `(a)` | 6.60x | 0.84x | 5.70x | 13.84x | +| `(a, b)` | 5.02x | 0.72x | 4.15x | 10.53x | +| `(a:)` | 9.83x | 0.83x | 5.94x | 17.24x | +| `(a:, b:)` | 7.66x | 0.70x | 4.66x | 16.24x | +| `(a, b:)` | 7.87x | 0.70x | 4.64x | 13.38x | +| `(a, *args)` | 2.08x | 0.78x | 1.57x | 3.58x | +| `(a:, **kwargs)` | 2.46x | 0.59x | 1.78x | 6.84x | +| `(a, *args, b:, **kwargs)` | 1.67x | 0.67x | 1.12x | 4.10x | \* `Dry::Core` [may cause incorrect behavior caused by hash collisions](https://github.com/dry-rb/dry-core/issues/63). -Results using Ruby 2.7.8 (either because these gems raise errors in Ruby 3.x, -or due to namespace collisions with more modern forks run against Ruby 3.x): - -|Method arguments|`DDMemoize` (1.0.0)|`Memery` (memery|`Memoist` (memoist|`Memoized` (1.1.1)|`Memoizer` (1.0.3)| -|--|--|--|--|--|--| -|`()` (none)|20.10x|3.79x|2.35x|22.22x|3.05x| -|`(a)`|17.03x|8.70x|12.39x|17.75x|11.06x| -|`(a, b)`|15.26x|8.41x|11.13x|15.27x|10.06x| -|`(a:)`|22.28x|12.52x|18.17x|20.59x|16.45x| -|`(a:, b:)`|10.09x|5.77x|9.15x|10.83x|11.99x| -|`(a, b:)`|20.01x|11.25x|16.88x|18.26x|15.34x| -|`(a, *args)`|2.98x|1.56x|2.27x|3.04x|1.96x| -|`(a:, **kwargs)`|2.73x|1.55x|2.20x|2.49x|2.10x| -|`(a, *args, b:, **kwargs)`|2.16x|1.16x|1.85x|1.99x|1.74x| +Results using Ruby 2.7.8 (because these gems raise errors in Ruby 3.x): + +| Method arguments | `ddmemoize` (1.0.0) | `memoist` (0.16.2) | `memoized` (1.1.1) | `memoizer` (1.0.3) | +|----------------------------|---------------------|--------------------|--------------------|--------------------| +| `()` (none) | 19.77x | 2.41x | 21.53x | 3.08x | +| `(a)` | 16.41x | 11.80x | 17.03x | 10.38x | +| `(a, b)` | 15.09x | 11.17x | 15.21x | 9.96x | +| `(a:)` | 20.91x | 17.19x | 18.48x | 15.66x | +| `(a:, b:)` | 20.48x | 17.30x | 18.74x | 16.18x | +| `(a, b:)` | 19.17x | 16.03x | 16.93x | 15.57x | +| `(a, *args)` | 2.91x | 2.08x | 2.90x | 1.81x | +| `(a:, **kwargs)` | 2.61x | 2.20x | 2.61x | 2.18x | +| `(a, *args, b:, **kwargs)` | 2.13x | 1.79x | 1.99x | 1.71x | + You can run benchmarks yourself with: diff --git a/benchmarks/Gemfile b/benchmarks/Gemfile index 5fbf778..8e96b6d 100644 --- a/benchmarks/Gemfile +++ b/benchmarks/Gemfile @@ -10,18 +10,23 @@ gem "benchmark", "0.3.0" gem "benchmark-ips", "2.14.0" gem "gem_bench", "2.0.1" +# NOTE: Regarding `require: false` below +# 1. GitHub version of MemoWise and the local source of MemoWise, share a namespace +# 2. memery & alt_memery share the namespace Memery +# 3. memoist & memoist3 share the namespace Memoist, and also share a load path for their version.rb files. +# This means we must `require: false` in `benchmarks/Gemfile` all, or all but one, of each of these duplicates, +# or we take care to only load them in discrete Ruby versions, +# to avoid a namespace collision before re-namespacing duplicates if RUBY_VERSION > "3" gem "alt_memery", "2.1.0", require: false gem "dry-core", "1.0.1" gem "memoist3", "1.0.0", require: false - gem "memoist2", "0.3.0" gem "memery", "1.6.0" else gem "ddmemoize", "1.0.0" gem "memoist", "0.16.2" - gem "memoist2", "0.3.0" gem "memoized", "1.1.1" gem "memoizer", "1.0.3" end -gem "memo_wise", github: "panorama-ed/memo_wise", branch: "main" +gem "memo_wise", github: "panorama-ed/memo_wise", branch: "main", require: false diff --git a/benchmarks/benchmarks.rb b/benchmarks/benchmarks.rb index 00080da..3745f5f 100644 --- a/benchmarks/benchmarks.rb +++ b/benchmarks/benchmarks.rb @@ -5,14 +5,17 @@ require "benchmark/ips" require "gem_bench/jersey" +# Constants used for temp file paths necessary to separate gem namespaces that would otherwise collide. GITHUB_MAIN = "MemoWise_GitHubMain" GITHUB_MAIN_BENCHMARK_NAME = "memo_wise-git-main" LOCAL_BENCHMARK_NAME = "memo_wise-local" -# Constants used for temp file paths necessary to separate gem namespaces that would otherwise collide. # 1. GitHub version of MemoWise and the local source of MemoWise, share a namespace # 2. memery & alt_memery share the namespace Memery -# 3. memoist, memoist2, & memoist3 share the namespace Memoist +# 3. memoist & memoist3 share the namespace Memoist, and also share a load path for their version.rb files. +# This means we must `require: false` in `benchmarks/Gemfile` all, or all but one, of each of these duplicates, +# or we take care to only load them in discrete Ruby versions, +# to avoid a namespace collision before re-namespacing duplicates re_namespaced_gems = [ GemBench::Jersey.new( gem_name: "memo_wise", @@ -43,9 +46,19 @@ activation_code: "extend MemoistThree", memoization_method: :memoize, }, - ) + ), + GemBench::Jersey.new( + gem_name: "memoist", + trades: { + "Memoist" => "MemoistOne" + }, + metadata: { + activation_code: "extend MemoistOne", + memoization_method: :memoize, + }, + ), ].each do |re_namespaced_gem| - re_namespaced_gem.doff_and_don + re_namespaced_gem.doff_and_don # Copies, re-namespaces, and requires each gem. end # Here we require memo_wise gem from the `pwd` source. @@ -54,18 +67,14 @@ require_relative "../lib/memo_wise" # Some gems do not yet work in Ruby 3 so we only require them if they're loaded -# in the Gemfile. -%w[memery memoist memoized memoizer ddmemoize dry-core]. +# in the Gemfile. Gems re-namespaced by GemBench::Jersey will have already been loaded by now. +%w[memery memoized memoizer ddmemoize dry-core]. each { |gem| require gem if Gem.loaded_specs.key?(gem) } -# alt_memery gem is a fork of memery, and uses the exact same namespace. -# This means we can't run them both at the same time, so we'll run memery on Ruby 2.7, and alt_memery on Ruby 3+. -# Similarly, memoist3 is a fork of memoist with Ruby 3 fixes. -# The Memoizable module from dry-core needs to be required manually. +# Some Gems Have Modules Which Need To Be Required Manually: +# 1. `dry-core` => Memoizable { # file_path => gem_name - "memery" => "alt_memery", - "memoist" => "memoist3", "dry/core/memoizable" => "dry-core", }. each { |file_path, gem_name| require file_path if Gem.loaded_specs.key?(gem_name) } @@ -116,16 +125,15 @@ def benchmark_name re_namespaced_gem.gem_name == "memo_wise" ? GITHUB_MAIN_BENCHMARK_NAME : re_namespaced_gem.gem_name, ) if re_namespaced_gem.required? }.concat( -[ + [ BenchmarkGem.new(MemoWise, "prepend MemoWise", :memo_wise, LOCAL_BENCHMARK_NAME), (BenchmarkGem.new(DDMemoize, "DDMemoize.activate(self)", :memoize, "ddmemoize") if defined?(DDMemoize)), (BenchmarkGem.new(Dry::Core, "include Dry::Core::Memoizable", :memoize, "dry-core") if defined?(Dry::Core)), (BenchmarkGem.new(Memery, "include Memery", :memoize, "memery") if defined?(Memery)), - (BenchmarkGem.new(Memoist, "extend Memoist", :memoize, "memoist") if defined?(Memoist)), - (BenchmarkGem.new(Memoist2, "include Memoist2", :memoize, "memoist2") if defined?(Memoist2)), (BenchmarkGem.new(Memoized, "include Memoized", :memoize, "memoized") if defined?(Memoized)), (BenchmarkGem.new(Memoizer, "include Memoizer", :memoize, "memoizer") if defined?(Memoizer)) - ])).compact.shuffle + ] + )).compact.shuffle puts "\nWill BENCHMARK_GEMS:\n\t#{BENCHMARK_GEMS.map(&:benchmark_name).join("\n\t")}\n" @@ -296,7 +304,7 @@ def positional_splat_keyword_and_double_splat_args(a, *args, b:, **kwargs) # Gem name is of the form: # "memoist (1.1.0): ()" # We use this mapping to get a header of the form - # "`memoist` (1.1.0) + # "`memoist` (1.1.0)" gem_name_parts = benchmark_gem["name"].split "`#{gem_name_parts[0]}` #{gem_name_parts[1][...-1]}" end.join("|")