Merge pull request #4112 from DataDog/tonycthsu/split-gemfiles

Split gemfiles for each runtime

.circleci/config.yml

@@ -82,10 +82,10 @@ step_appraisal_install: &step_appraisal_install
     name: Install Appraisal gems
     command: |
       if [ "$CI_BUNDLE_CACHE_HIT" != 1 ]; then
-        bundle exec appraisal generate
+        bundle exec rake dependency:generate
         bundle exec rake dependency:install
       else
-        bundle exec appraisal generate
+        bundle exec rake dependency:generate # Generate the appraisal files to match the lockfiles in the tree
         echo "All required gems were found in cache."
       fi
 step_compute_bundle_checksum: &step_compute_bundle_checksum

.github/workflows/lock-dependency.yml

@@ -98,9 +98,7 @@ jobs:
           gem -v
           bundler -v
       - run: bundle install
-
-      # TODO: Migrate away from `appraisal`
-      - run: bundle exec appraisal generate
+      - run: bundle exec rake dependency:generate
       - run: bundle exec rake dependency:lock
       - uses: actions/upload-artifact@v4
         with:

.gitignore

@@ -44,6 +44,7 @@ ext/**/skipped_reason.txt
 # lock files
 Gemfile.lock
 Gemfile-*.lock
+*.gemfile.lock
 
 # bundle config
 gemfiles/.bundle

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "files.associations": {
+    "*.gemfile": "ruby",
     "Dockerfile*": "dockerfile"
   }
 }

Appraisals

@@ -1,3 +1,7 @@
+# Do not edit this file directly.
+#
+# The dependency management has been migrated to `tasks/dependency.rake`
+
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'datadog/version'

Gemfile

@@ -1,94 +1 @@
-source 'https://rubygems.org'
-
-gemspec
-
-gem 'appraisal', '~> 2.4.0'
-gem 'benchmark-ips', '~> 2.8'
-gem 'benchmark-memory', '< 0.2' # V0.2 only works with 2.5+
-
-gem 'climate_control', '~> 0.2.0'
-
-gem 'concurrent-ruby'
-gem 'extlz4', '~> 0.3', '>= 0.3.3' if RUBY_PLATFORM != 'java' # Used to test lz4 compression done by libdatadog
-gem 'json-schema', '< 3' # V3 only works with 2.5+
-gem 'memory_profiler', '~> 0.9'
-
-gem 'os', '~> 1.1'
-gem 'pimpmychangelog', '>= 0.1.2'
-gem 'pry'
-if RUBY_PLATFORM != 'java'
-  # There's a few incompatibilities between pry/pry-byebug on older Rubies
-  # There's also a few temproary incompatibilities with newer rubies
-  gem 'pry-byebug' if RUBY_VERSION >= '2.6.0' && RUBY_ENGINE != 'truffleruby' && RUBY_VERSION < '3.2.0'
-  gem 'pry-nav' if RUBY_VERSION < '2.6.0'
-  gem 'pry-stack_explorer'
-else
-  gem 'pry-debugger-jruby'
-end
-gem 'rake', '>= 10.5'
-gem 'rake-compiler', '~> 1.1', '>= 1.1.1' # To compile native extensions
-gem 'rspec', '~> 3.13'
-gem 'rspec-collection_matchers', '~> 1.1'
-gem 'rspec-wait', '~> 0'
-
-gem 'rspec_junit_formatter', '>= 0.5.1'
-
-# Merging branch coverage results does not work for old, unsupported rubies and JRuby
-# We have a fix up for review, https://github.com/simplecov-ruby/simplecov/pull/972,
-# but given it only affects unsupported version of Ruby, it might not get merged.
-gem 'simplecov', git: 'https://github.com/DataDog/simplecov', ref: '3bb6b7ee58bf4b1954ca205f50dd44d6f41c57db'
-gem 'simplecov-cobertura', '~> 2.1.0' # Used by codecov
-
-gem 'warning', '~> 1' # NOTE: Used in spec_helper.rb
-gem 'webmock', '>= 3.10.0'
-
-gem 'rexml', '>= 3.2.7' # https://www.ruby-lang.org/en/news/2024/05/16/dos-rexml-cve-2024-35176/
-
-if RUBY_VERSION.start_with?('3.4.')
-  # ruby 3.4 breaks stable webrick; we need this fix until a version later than 1.8.1 comes out
-  gem 'webrick', git: 'https://github.com/ruby/webrick.git', ref: '0c600e169bd4ae267cb5eeb6197277c848323bbe'
-elsif RUBY_VERSION >= '3.0.0' # No longer bundled by default since Ruby 3.0
-  gem 'webrick', '>= 1.7.0'
-end
-
-if RUBY_VERSION >= '2.6.0'
-  # 1.50 is the last version to support Ruby 2.6
-  gem 'rubocop', '~> 1.50.0', require: false
-  gem 'rubocop-packaging', '~> 0.5.2', require: false
-  gem 'rubocop-performance', '~> 1.9', require: false
-  # 2.20 is the last version to support Ruby 2.6
-  gem 'rubocop-rspec', ['~> 2.20', '< 2.21'], require: false
-end
-
-# Optional extensions
-# TODO: Move this to Appraisals?
-# dogstatsd v5, but lower than 5.2, has possible memory leak with datadog.
-# @see https://github.com/DataDog/dogstatsd-ruby/issues/182
-gem 'dogstatsd-ruby', '>= 3.3.0', '!= 5.0.0', '!= 5.0.1', '!= 5.1.0'
-
-# Profiler testing dependencies
-# NOTE: We're excluding versions 3.7.0 and 3.7.1 for the reasons documented in #1424.
-#       Since most of our customers won't have BUNDLE_FORCE_RUBY_PLATFORM=true, it's not something we want to add
-#       to our CI, so we just shortcut and exclude specific versions that were affecting our CI.
-if RUBY_PLATFORM != 'java'
-  if RUBY_VERSION >= '2.7.0' # Bundler 1.x fails to find that versions >= 3.8.0 are not compatible because of binary gems
-    gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1']
-  else
-    gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1', '< 3.19.2']
-  end
-end
-
-group :check do
-  if RUBY_VERSION >= '3.0.0' && RUBY_PLATFORM != 'java'
-    gem 'rbs', '~> 3.6', require: false
-    gem 'steep', '~> 1.7.0', require: false
-  end
-  gem 'ruby_memcheck', '>= 3' if RUBY_VERSION >= '3.4.0' && RUBY_PLATFORM != 'java'
-  gem 'standard', require: false
-end
-
-group :dev do
-  gem 'ruby-lsp', require: false if RUBY_VERSION >= '3.0.0' && RUBY_PLATFORM != 'java'
-end
-
-gem 'uri', '~> 0.13.1' if RUBY_VERSION.start_with? '3'
+eval_gemfile("#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION.split('.').take(2).join('.')}.gemfile")

appraisal/generate.rb

@@ -0,0 +1,88 @@
+# Ruby script to replace `bundle exec appraisal generate`
+#
+# Why???
+#
+# 1. `Appraisals` file is extremely hard to extend, because the definition is evaluated by `instance_eval`
+# 2. Not all `Bundler::DSL` methods are supported.
+#
+# This implementation provides a much easier interface to customize our needs,
+# while still using the same Appraisal formatting.
+#
+# For example, it solves the incompatbility of `eval_gemfile` from `Bundler::DSL`
+#
+# Usage: `bundle exec ruby appraisal/generate.rb`
+
+require "appraisal/appraisal"
+
+require_relative "../tasks/appraisal_conversion"
+
+gemfile = Appraisal::Gemfile.new.tap do |g|
+  # Support `eval_gemfile` for `Bundler::DSL`
+  g.define_singleton_method(:eval_gemfile) {|file| load(file) }
+  g.load(ENV["BUNDLE_GEMFILE"] || "Gemfile")
+end
+
+appraisals = []
+
+REMOVED_GEMS = {
+  :check => [
+    'rbs',
+    'steep',
+    'standard',
+  ],
+  :dev => [
+    'ruby-lsp',
+  ],
+}
+
+define_singleton_method(:appraise) do |name, &block|
+  # Customize name
+  name = "#{AppraisalConversion.runtime_identifier}_#{name}"
+  appraisal = Appraisal::Appraisal.new(name, gemfile)
+  appraisal.instance_eval(&block)
+  # Customize callback for removal
+  REMOVED_GEMS.each do |group_name, gems|
+    appraisal.group(group_name) do
+      gems.each { |gem_name| remove_gem gem_name }
+    end
+  end
+  appraisals << appraisal
+end
+
+# Builds a matrix of versions to test for a given integration
+
+# `range`: the range of versions to test
+# `gem`  : optional, gem name to test (gem name can be different from the integration name)
+# `min`  : optional, minimum version to test
+# `meta` : optional, additional metadata (development dependencies, etc.) for the group
+def build_coverage_matrix(integration, range, gem: nil, min: nil, meta: {})
+  gem ||= integration
+
+  if min
+    appraise "#{integration}-min" do
+      gem gem, "= #{min}"
+      meta.each { |k, v| v ? gem(k, v) : gem(k) }
+    end
+  end
+
+  range.each do |n|
+    appraise "#{integration}-#{n}" do
+      gem gem, "~> #{n}"
+      meta.each { |k, v| v ? gem(k, v) : gem(k) }
+    end
+  end
+
+  appraise "#{integration}-latest" do
+    # The latest group declares dependencies without version constraints,
+    # still requires being updated to pick up the next major version and
+    # committing the changes to lockfiles.
+    gem gem
+    meta.each { |k, v| v ? gem(k, v) : gem(k) }
+  end
+end
+
+load(AppraisalConversion.definition)
+
+puts appraisals.map(&:name)
+
+appraisals.each(&:write_gemfile)

docker-compose.yml

@@ -16,7 +16,7 @@ services:
       - testagent
     env_file: ./.env
     environment: &common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-2.5
+      BUNDLE_GEMFILE: /app/ruby-2.5.gemfile
       DD_AGENT_HOST: testagent
       DD_TRACE_AGENT_PORT: 9126
       TEST_DATADOG_INTEGRATION: 1
@@ -45,7 +45,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-2.6
+      BUNDLE_GEMFILE: /app/ruby-2.6.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -61,7 +61,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-2.7
+      BUNDLE_GEMFILE: /app/ruby-2.7.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -77,7 +77,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-3.0
+      BUNDLE_GEMFILE: /app/ruby-3.0.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -93,7 +93,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-3.1
+      BUNDLE_GEMFILE: /app/ruby-3.1.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -109,7 +109,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-3.2
+      BUNDLE_GEMFILE: /app/ruby-3.2.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -125,7 +125,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-3.3
+      BUNDLE_GEMFILE: /app/ruby-3.3.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -141,7 +141,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-3.4
+      BUNDLE_GEMFILE: /app/ruby-3.4.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -158,7 +158,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-jruby-9.2
+      BUNDLE_GEMFILE: /app/jruby-9.2.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -173,7 +173,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-jruby-9.3
+      BUNDLE_GEMFILE: /app/jruby-9.3.gemfile
     stdin_open: true
     tty: true
     volumes:
@@ -188,7 +188,7 @@ services:
     env_file: ./.env
     environment:
       <<: *common-environment
-      BUNDLE_GEMFILE: /app/Gemfile-jruby-9.4
+      BUNDLE_GEMFILE: /app/jruby-9.4.gemfile
     stdin_open: true
     tty: true
     volumes: