From 49ecf3d4c898b6b71764d6bd1e819c13bb5af186 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 20 Dec 2023 14:13:31 -0500 Subject: [PATCH] change `-linux` to `-linux-gnu` everywhere Test gems built with the `-linux-gnu` suffix only on ruby 3.1 and later, which are the ones that ship with rubygems 3.2.33 or later (the version that first recognizes the `-gnu` suffix). Also make sure we tag the `-linux-gnu` docker images as `-linux` for backwards compatibility, and make sure we test that workflow. --- .github/workflows/ci.yml | 48 ++++++++++++++++++++++------ .github/workflows/publish-images.yml | 12 +++---- .gitignore | 9 +++--- Dockerfile.mri.erb | 8 ++--- History.md | 3 ++ README.md | 33 ++++++++++++------- Rakefile | 18 +++++++---- lib/rake_compiler_dock.rb | 3 +- test/rcd_test/Rakefile | 16 +++++++++- test/test_starter.rb | 2 +- 10 files changed, 106 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1478f1a3..e1e370bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,18 +45,22 @@ jobs: fail-fast: false matrix: include: - - platform: aarch64-linux - - platform: arm-linux + - platform: aarch64-linux-gnu + alias: aarch64-linux + - platform: arm-linux-gnu + alias: arm-linux - platform: arm64-darwin - platform: jruby - platform: x64-mingw-ucrt static: true - platform: x64-mingw32 static: true - - platform: x86-linux + - platform: x86-linux-gnu + alias: x86-linux - platform: x86-mingw32 - platform: x86_64-darwin - - platform: x86_64-linux + - platform: x86_64-linux-gnu + alias: x86_64-linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -114,6 +118,21 @@ jobs: name: gem-${{ matrix.platform }}-static path: test/rcd_test/pkg/*-*-*.gem + - if: matrix.alias + name: Build native gem ${{ matrix.alias }} + run: | + cd test/rcd_test/ + bundle install + bundle exec rake clean clobber + bundle exec rake gem:${{ matrix.alias }} + + - if: matrix.alias + name: Upload native gem ${{ matrix.alias }} + uses: actions/upload-artifact@v3 + with: + name: gem-${{ matrix.alias }} + path: test/rcd_test/pkg/*-*-*.gem + test_source_gem: name: test source needs: build_source_gem @@ -172,6 +191,15 @@ jobs: platform: x86_64-darwin - os: ubuntu platform: x86_64-linux + - os: ubuntu # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + ruby: "3.1" + platform: x86_64-linux-gnu + - os: ubuntu + ruby: "3.2" + platform: x86_64-linux-gnu + - os: ubuntu + ruby: "3.3" + platform: x86_64-linux-gnu - os: ubuntu ruby: jruby platform: jruby @@ -282,22 +310,22 @@ jobs: matrix: include: - from_image: amd64/centos - platform: x86_64-linux + platform: x86_64-linux # centos-8 ships ruby 2.5, rubygems won't recognize -gnu suffix dockerfile: centos - from_image: navikey/raspbian-bullseye - platform: arm-linux + platform: arm-linux # bullseye ships ruby 2.7, rubygems won't recognize -gnu suffix dockerfile: debian - from_image: arm64v8/ubuntu - platform: aarch64-linux + platform: aarch64-linux # arm64v8 ships ruby 3.0, rubygems won't recognize -gnu suffix dockerfile: debian - from_image: i386/alpine - platform: x86-linux + platform: x86-linux-gnu dockerfile: alpine - from_image: arm32v6/alpine - platform: arm-linux + platform: arm-linux-gnu dockerfile: alpine - from_image: alpine - platform: x86_64-linux + platform: x86_64-linux-gnu dockerfile: alpine runs-on: ubuntu-latest steps: diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index ca5ff384..6bacffba 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -13,15 +13,15 @@ jobs: fail-fast: false matrix: platform: - - x86-mingw32 + - aarch64-linux-gnu + - arm-linux-gnu + - arm64-darwin - x64-mingw-ucrt - x64-mingw32 - - x86-linux - - x86_64-linux + - x86-linux-gnu + - x86-mingw32 - x86_64-darwin - - arm64-darwin - - arm-linux - - aarch64-linux + - x86_64-linux-gnu - jruby runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 09c077a2..e9bdd195 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,14 @@ /.bundle/ /.yardoc -/Dockerfile.mri.aarch64-linux -/Dockerfile.mri.arm-linux +/Dockerfile.mri.aarch64-linux-gnu +/Dockerfile.mri.arm-linux-gnu /Dockerfile.mri.arm64-darwin -/Dockerfile.mri.arm64-linux /Dockerfile.mri.x64-mingw-ucrt /Dockerfile.mri.x64-mingw32 -/Dockerfile.mri.x86-linux +/Dockerfile.mri.x86-linux-gnu /Dockerfile.mri.x86-mingw32 /Dockerfile.mri.x86_64-darwin -/Dockerfile.mri.x86_64-linux +/Dockerfile.mri.x86_64-linux-gnu /Gemfile.lock /_yardoc/ /cache/ diff --git a/Dockerfile.mri.erb b/Dockerfile.mri.erb index 7a6d1e90..137b317e 100644 --- a/Dockerfile.mri.erb +++ b/Dockerfile.mri.erb @@ -1,7 +1,7 @@ <% image = case platform - when /x86_64-linux/ then "quay.io/pypa/manylinux2014_x86_64" - when /x86-linux/ then "quay.io/pypa/manylinux2014_i686" + when /x86_64-linux-gnu/ then "quay.io/pypa/manylinux2014_x86_64" + when /x86-linux-gnu/ then "quay.io/pypa/manylinux2014_i686" else "ubuntu:20.04" end manylinux = !!(image =~ /manylinux/) @@ -68,8 +68,8 @@ RUN dpkg -i /debs/*.deb RUN apt-get -y update && \ apt-get install -y <% if platform =~ /darwin/ %> clang python lzma-dev libxml2-dev libssl-dev libc++-10-dev <% end %><% -if platform =~ /aarch64-linux/ %> gcc-aarch64-linux-gnu g++-aarch64-linux-gnu <% end %><% -if platform =~ /arm-linux/ %> gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf <% end %><% +if platform =~ /aarch64-linux-gnu/ %> gcc-aarch64-linux-gnu g++-aarch64-linux-gnu <% end %><% +if platform =~ /arm-linux-gnu/ %> gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf <% end %><% if platform =~ /x86-mingw32/ %> gcc-mingw-w64-i686 g++-mingw-w64-i686 <% end %><% if platform =~ /x64-mingw32/ %> gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 <% end %> && \ rm -rf /var/lib/apt/lists/* diff --git a/History.md b/History.md index 8dab9ee5..608b0d7c 100644 --- a/History.md +++ b/History.md @@ -1,6 +1,9 @@ next / unreleased ----------------- +* Linux builds and their images are now fully qualified with the libc flavor. So, `x86_64-linux` + should now be referred to as `x86_64-linux-gnu`, and the generated files are also named with the + libc name. * Replace `rvm` with `rbenv` and `ruby-build` - `rvm` has been replaced by `rbenv` and `ruby-build` - no longer applying sendfile patches to bootstrap rubies diff --git a/README.md b/README.md index c28b8de0..a0ac8c72 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Your Rakefile should enable cross compilation like so: ```ruby exttask = Rake::ExtensionTask.new('my_extension', my_gem_spec) do |ext| ext.cross_compile = true - ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux x86_64-darwin arm64-darwin] + ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux-gnu x86_64-linux-gnu x86_64-darwin arm64-darwin] end ``` @@ -44,7 +44,7 @@ Additionally it may also be used to build ffi based binary gems like [libusb](ht ### Interactive Usage Rake-compiler-dock offers the shell command `rake-compiler-dock` and a [ruby API](http://www.rubydoc.info/gems/rake-compiler-dock/RakeCompilerDock) for issuing commands within the docker image, described below. -There are dedicated images for `x86-mingw32`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux`, `x86_64-linux`, `x86_64-darwin`, `arm64-darwin` and `jruby` targets. +There are dedicated images for targets: `aarch64-linux-gnu`, `arm-linux-gnu`, `arm64-darwin`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux-gnu`, `x86-mingw32`, `x86_64-darwin`, `x86_64-linux-gnu`, and `jruby`. The images contain all supported cross ruby versions, with the exception of `x64-mingw32`, which has versions before 3.1 only, and `x64-mingw-ucrt`, which has only ruby-3.1+. This is to match the [changed platform of RubyInstaller-3.1](https://rubyinstaller.org/2021/12/31/rubyinstaller-3.1.0-1-released.html). @@ -68,12 +68,12 @@ To build x86 Windows and x86_64 Linux binary gems interactively, it can be calle user@host:$ ls pkg/*.gem your-gem-1.0.0.gem your-gem-1.0.0-x86-mingw32.gem - user@host:$ RCD_PLATFORM=x86_64-linux rake-compiler-dock # this enters a container for amd64 Linux target + user@host:$ RCD_PLATFORM=x86_64-linux-gnu rake-compiler-dock # this enters a container for amd64 Linux target user@adc55b2b92a9:$ bundle user@adc55b2b92a9:$ rake cross native gem user@adc55b2b92a9:$ exit user@host:$ ls pkg/*.gem - your-gem-1.0.0.gem your-gem-1.0.0-x86_64-linux.gem + your-gem-1.0.0.gem your-gem-1.0.0-x86_64-linux-gnu.gem Or non-interactive: @@ -118,10 +118,21 @@ To make the build process reproducible for other parties, it is recommended to a This can be done like this: ```ruby +PLATFORMS = %w[ + aarch64-linux-gnu + arm-linux-gnu + arm64-darwin + x64-mingw-ucrt + x64-mingw32 + x86-linux-gnu + x86-mingw32 + x86_64-darwin + x86_64-linux-gnu +] task 'gem:native' do require 'rake_compiler_dock' sh "bundle package --all" # Avoid repeated downloads of gems by using gem files from the host. - %w[ x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux arm-linux aarch64-linux x86_64-darwin arm64-darwin ].each do |plat| + PLATFORMS.each do |plat| RakeCompilerDock.sh "bundle --local && rake native:#{plat} gem", platform: plat end RakeCompilerDock.sh "bundle --local && rake java gem", rubyvm: :jruby @@ -187,10 +198,10 @@ jobs: name: "native-gem" runs-on: ubuntu-latest container: - image: "ghcr.io/rake-compiler/rake-compiler-dock-image:1.2.2-mri-x86_64-linux" + image: "ghcr.io/rake-compiler/rake-compiler-dock-image:1.2.2-mri-x86_64-linux-gnu" steps: - uses: actions/checkout@v2 - - run: bundle install && bundle exec rake gem:x86_64-linux:rcd + - run: bundle install && bundle exec rake gem:x86_64-linux-gnu:rcd - uses: actions/upload-artifact@v2 with: name: native-gem @@ -198,10 +209,10 @@ jobs: retention-days: 1 ``` -Where the referenced rake task might be defined by: +Where the referenced rake task might be defined by something like: ``` ruby -cross_platforms = ["x64-mingw32", "x86_64-linux", "x86_64-darwin", "arm64-darwin"] +cross_platforms = ["x64-mingw32", "x86_64-linux-gnu", "x86_64-darwin", "arm64-darwin"] namespace "gem" do cross_platforms.each do |platform| @@ -223,7 +234,7 @@ For an example of rake tasks that support this style of invocation, visit https: OCI images snapshotted from `main` are published weekly to Github Container Registry with the string "snapshot" in place of the version number in the tag name, e.g.: -- `ghcr.io/rake-compiler/rake-compiler-dock-image:snapshot-mri-x86_64-linux` +- `ghcr.io/rake-compiler/rake-compiler-dock-image:snapshot-mri-x86_64-linux-gnu` These images are intended for integration testing. They may not work properly and should not be considered production ready. @@ -237,7 +248,7 @@ The following variables are recognized by rake-compiler-dock: * `RCD_RUBYVM` - The ruby VM and toolchain to be used. Must be one of `mri`, `jruby`. * `RCD_PLATFORM` - The target rubygems platform. - Must be a space separated list out of `x86-mingw32`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux`, `x86_64-linux`, `arm-linux`, `aarch64-linux`, `x86_64-darwin` and `arm64-darwin`. + Must be a space separated list out of `aarch64-linux-gnu`, `arm-linux-gnu`, `arm64-darwin`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux-gnu`, `x86-mingw32`, `x86_64-darwin`, `x86_64-linux-gnu`. It is ignored when `rubyvm` is set to `:jruby`. * `RCD_IMAGE` - The docker image that is downloaded and started. Defaults to "ghcr.io/rake-compiler/rake-compiler-dock-image:IMAGE_VERSION-PLATFORM" with an image version that is determined by the gem version. diff --git a/Rakefile b/Rakefile index f79bbea8..c420370e 100644 --- a/Rakefile +++ b/Rakefile @@ -9,15 +9,16 @@ CLEAN.include("tmp") RakeCompilerDock::GemHelper.install_tasks platforms = [ - ["x86-mingw32", "i686-w64-mingw32"], - ["x64-mingw32", "x86_64-w64-mingw32"], + # tuple is [platform, target] + ["aarch64-linux-gnu", "aarch64-linux-gnu"], + ["arm-linux-gnu", "arm-linux-gnueabihf"], + ["arm64-darwin", "aarch64-apple-darwin"], ["x64-mingw-ucrt", "x86_64-w64-mingw32"], - ["x86-linux", "i686-redhat-linux"], - ["x86_64-linux", "x86_64-redhat-linux"], + ["x64-mingw32", "x86_64-w64-mingw32"], + ["x86-linux-gnu", "i686-redhat-linux-gnu"], + ["x86-mingw32", "i686-w64-mingw32"], ["x86_64-darwin", "x86_64-apple-darwin"], - ["arm64-darwin", "aarch64-apple-darwin"], - ["arm-linux", "arm-linux-gnueabihf"], - ["aarch64-linux", "aarch64-linux-gnu"], + ["x86_64-linux-gnu", "x86_64-redhat-linux-gnu"], ] namespace :build do @@ -30,6 +31,9 @@ namespace :build do task sdf do image_name = RakeCompilerDock::Starter.container_image_name(platform: platform) sh(*RakeCompilerDock.docker_build_cmd(platform), "-t", image_name, "-f", "Dockerfile.mri.#{platform}", ".") + if image_name.include?("linux-gnu") + sh("docker", "tag", image_name, image_name.sub("linux-gnu", "linux")) + end end df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding) diff --git a/lib/rake_compiler_dock.rb b/lib/rake_compiler_dock.rb index 31478645..f73d7e26 100644 --- a/lib/rake_compiler_dock.rb +++ b/lib/rake_compiler_dock.rb @@ -26,7 +26,8 @@ module RakeCompilerDock # # Option +:platform+ can be set to a list of space separated values. # It selects the docker image(s) with an appropriate toolchain. - # Allowed values are +:x86-mingw32+, +x64-mingw32+, +x86-linux+ or +x86_64-linux+. + # Allowed values are +aarch64-linux-gnu+, +arm-linux-gnu+, +arm64-darwin+, +x64-mingw-ucrt+, + # +x64-mingw32+, +x86-linux-gnu+, +x86-mingw32+, +x86_64-darwin+, +x86_64-linux-gnu+. # If the list contains multiple values, +cmd+ is consecutively executed in each of the docker images, # Option +:platform+ is ignored when +:rubyvm+ is set to +:jruby+. # Default is "x86-mingw32 x64-mingw32" . diff --git a/test/rcd_test/Rakefile b/test/rcd_test/Rakefile index 97b798a2..e4b0d63a 100644 --- a/test/rcd_test/Rakefile +++ b/test/rcd_test/Rakefile @@ -18,7 +18,21 @@ else ext.ext_dir = 'ext/mri' ext.lib_dir = 'lib/rcd_test' ext.cross_compile = true - ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux aarch64-linux arm-linux x86_64-darwin arm64-darwin] + ext.cross_platform = %w[ + aarch64-linux + aarch64-linux-gnu + arm-linux + arm-linux-gnu + arm64-darwin + x64-mingw-ucrt + x64-mingw32 + x86-linux + x86-linux-gnu + x86-mingw32 + x86_64-darwin + x86_64-linux + x86_64-linux-gnu + ] end end diff --git a/test/test_starter.rb b/test/test_starter.rb index b3ad9ae9..70ac29ee 100644 --- a/test/test_starter.rb +++ b/test/test_starter.rb @@ -113,7 +113,7 @@ def test_container_jrubyvm? assert(Starter.container_jrubyvm?({:rubyvm => "jruby"})) assert(Starter.container_jrubyvm?({:platform => "jruby"})) refute(Starter.container_jrubyvm?({:rubyvm => "mri"})) - refute(Starter.container_jrubyvm?({:platform => "x86_64-linux"})) + refute(Starter.container_jrubyvm?({:platform => "x86_64-linux-gnu"})) end def test_platforms