From afb4f9acbce56adbc4bd17e82a6b5cad61db0db2 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 23:24:05 +0100 Subject: [PATCH 1/3] Explicitly specify target for Darwin Intel build By default uses the target of the system where the script is executed. If run darwin build on MacOS M1 laptop, it generates 2 arm binaries. And command `lito` returns error that both files have same arch. Reproduction of the problem: ``` $ llvm-config --host-target arm64-apple-darwin23.0.0 $ make crystal stats=true release=true Using /usr/local/bin/llvm-config [version= 15.0.7] CRYSTAL_CONFIG_BUILD_COMMIT="1f592eca6" CRYSTAL_CONFIG_PATH='$ORIGIN/../share/crystal/src' SOURCE_DATE_EPOCH="1699639716" CRYSTAL_CONFIG_LIBRARY_PATH= ./bin/crystal build --no-debug -D strict_multi_assign .... $ file .../crystal/embedded/bin/crystal .../crystal/embedded/bin/crystal: Mach-O 64-bit executable arm64 ``` --- omnibus/config/software/crystal.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index 4a31e3cb..f30d79ea 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -58,8 +58,9 @@ copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" # Compile for Intel - command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + command "make crystal stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env move output_bin, "#{output_bin}_x86_64" + block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } # Clean up make "clean_cache clean", env: env @@ -77,6 +78,7 @@ command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env delete "#{output_path}/crystal.o" + block { raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64") } # Lipo them up command "lipo -create -output #{output_bin} #{output_bin}_x86_64 #{output_bin}_arm64" From e5a56584126fd9b3c5de07364566224937076c0f Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 10 Nov 2023 23:42:50 +0100 Subject: [PATCH 2/3] Update CXXFLAGS to have target argument before build --- omnibus/config/software/crystal.rb | 84 ++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index f30d79ea..76a51523 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -1,17 +1,13 @@ CRYSTAL_VERSION = ENV['CRYSTAL_VERSION'] CRYSTAL_SHA1 = ENV['CRYSTAL_SHA1'] FIRST_RUN = ENV["FIRST_RUN"] -CRYSTAL_SRC = (ENV['CRYSTAL_SRC'] || "").strip +CRYSTAL_SRC = ENV.fetch('CRYSTAL_SRC', 'https://github.com/crystal-lang/crystal') name "crystal" default_version CRYSTAL_VERSION skip_transitive_dependency_licensing true -if CRYSTAL_SRC.empty? - source git: "https://github.com/crystal-lang/crystal" -else - source git: CRYSTAL_SRC -end +source git: CRYSTAL_SRC dependency "pcre2" dependency "bdw-gc" @@ -46,42 +42,84 @@ end build do - command "git checkout #{CRYSTAL_SHA1}", cwd: project_dir + block { puts "\n=== Starting a build phase for crystal ===\n\n" } + + command "git checkout '#{CRYSTAL_SHA1}'", cwd: project_dir + + # Native crystal binary with cross-platform functionality + block { puts "\n=== Build crystal's deps in #{project_dir}\n\n" } mkdir "#{project_dir}/deps" - make "deps", env: env + make "deps", env: env.dup mkdir ".build" - command "echo #{Dir.pwd}", env: env - - crflags = "--no-debug" + block { puts "\n=== Build native crystal bin with embedded universal crystal binary\n\n" } copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" - - # Compile for Intel - command "make crystal stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env - move output_bin, "#{output_bin}_x86_64" - block { raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") } + command ".build/crystal --version", env: env.dup + command "file .build/crystal", env: env.dup + + crflags = "--no-debug" + command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup + block "Testing the result file" do + puts "===> Testing the result file #{output_bin}" + raise "Could not build native crystal binary: #{output_bin}" unless File.exist?("#{output_bin}") + end + command "file #{output_bin}", env: env.dup + command "#{output_bin} --version", env: env.dup + + # Restore compiler w/ cross-compile support + block { puts "\n\n=== Restore compiler with cross-compile support ===\n\n" } + move "#{output_bin}", ".build/crystal" + + make "clean_cache clean", env: env + + # x86_64 crystal binary + block { puts "\n\n=== Building x86_64 binary ===\n\n" } + + original_CXXFLAGS_env = env["CXXFLAGS"].dup + original_LDFLAGS_env = env["LDFLAGS"].dup + + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" + env["LDFLAGS"] = original_LDFLAGS_env + " -v -target x86_64-apple-darwin" + env["LDLIBS"] = "-v -target x86_64-apple-darwin" + make "deps", env: env.dup + + make "crystal verbose=true stats=true release=true target=x86_64-apple-darwin CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + command "clang #{output_path}/crystal.o -o #{output_bin}_x86_64 -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + block "Testing the result file" do + puts "===> Testing the result file #{output_bin}_x86_64" + raise "Could not build crystal x86_64" unless File.exist?("#{output_bin}_x86_64") + end + # NOTE: Add validation of the output + command "file #{output_bin}_x86_64", env: env # Clean up make "clean_cache clean", env: env - # Restore x86_64 compiler w/ cross-compile support - mkdir ".build" - copy "#{output_bin}_x86_64", ".build/crystal" - + # arm64 crystal binary + block { puts "\n\n=== Building arm64 version ===\n\n" } + # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools - env["CXXFLAGS"] << " -target arm64-apple-darwin" + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" make "deps", env: env make "crystal stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env - command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + + block "Testing the result file" do + puts "===> Testing the result file #{output_bin}_arm64" + raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64") + end + # NOTE: Add validation of the output + command "file #{output_bin}_arm64", env: env delete "#{output_path}/crystal.o" - block { raise "Could not build crystal arm64" unless File.exist?("#{output_bin}_arm64") } # Lipo them up + block { puts "\n\n=== Combine x86_64 and arm64 binaries in single universal binary ===\n\n" } command "lipo -create -output #{output_bin} #{output_bin}_x86_64 #{output_bin}_arm64" + + # Clean up delete "#{output_bin}_x86_64" delete "#{output_bin}_arm64" From 075c9164735cad0aaa59a7c2132c992fd9c92e75 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Thu, 4 Jan 2024 23:13:47 +0100 Subject: [PATCH 3/3] cherry pick #266 --- darwin/Makefile | 18 +++++++++--------- omnibus/config/software/crystal.rb | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/darwin/Makefile b/darwin/Makefile index 584a93ef..4986d033 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -9,7 +9,7 @@ CRYSTAL_SHA1 ?= $(CRYSTAL_VERSION) ## Git tag/branch/sha1 to checkout and build PACKAGE_ITERATION ?= 1 FORCE_GIT_TAGGED ?= 1 ## Require build to be based on git tag/branch -PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-x86_64.tar.gz +PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-universal.tar.gz. Eg: https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz OUTPUT_DIR = build @@ -40,15 +40,15 @@ help: ## Show this help awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf " \033[36m%s\033[0m\n", $$2}; /^## / {printf " %s\n", $$2}' .PHONY: darwin-previous -darwin-previous: $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal ## download previous crystal darwin release +darwin-previous: $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal ## download previous crystal darwin release # Once there are prior builds for arm64, this can use DARWIN_ARCH -$(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal: - curl -L -o /tmp/crystal-darwin-x86_64.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ - && mkdir -p $(CURDIR)/../omnibus/crystal-darwin-x86_64 \ - && tar xfz /tmp/crystal-darwin-x86_64.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin-x86_64 --strip-components=1 \ - && rm /tmp/crystal-darwin-x86_64.tar.gz \ - && chmod +x $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal +$(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal: + curl -L -o /tmp/crystal-darwin-universal.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ + && mkdir -p $(CURDIR)/../omnibus/crystal-darwin-universal \ + && tar xfz /tmp/crystal-darwin-universal.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin-universal --strip-components=1 \ + && rm /tmp/crystal-darwin-universal.tar.gz \ + && chmod +x $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) @@ -75,6 +75,6 @@ clean: ## Clean up build directory rm -Rf $(CURDIR)/tmp rm -Rf $(CURDIR)/../omnibus/pkg/crystal-* rm -Rf $(CURDIR)/../omnibus/pkg/version-* - rm -Rf $(CURDIR)/../omnibus/crystal-darwin-* + rm -Rf $(CURDIR)/../omnibus/crystal-darwin-universal rm -Rf /var/cache/omnibus/* rm -Rf /opt/crystal/* diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index 76a51523..5330ff41 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -54,7 +54,7 @@ mkdir ".build" block { puts "\n=== Build native crystal bin with embedded universal crystal binary\n\n" } - copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" + copy "#{Dir.pwd}/crystal-darwin-universal/embedded/bin/crystal", ".build/crystal" command ".build/crystal --version", env: env.dup command "file .build/crystal", env: env.dup