From 5fb83e306e3041e87286f03746ec31b6c136f912 Mon Sep 17 00:00:00 2001 From: Gabriel Gerlero Date: Wed, 27 Dec 2023 23:48:35 -0300 Subject: [PATCH] Update dependency bundling --- .github/workflows/build-test.yml | 56 ++-------------- Makefile | 108 +++++++++---------------------- README.md | 2 +- bundle_deps.py | 62 ++++++++++++++++++ macho.py | 18 ++++++ relativize_install_names.py | 11 ++-- 6 files changed, 124 insertions(+), 133 deletions(-) create mode 100755 bundle_deps.py create mode 100644 macho.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index bd8a6bf..2467729 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -78,54 +78,18 @@ env: OPENFOAM: ${{ inputs.openfoam-version || inputs.openfoam-git-branch }} jobs: - deps: + build: runs-on: ${{ inputs.build-os || 'macos-12' }} env: BUILD_OS: ${{ inputs.build-os || 'macos-12' }} - outputs: - deps-restore-key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }} - build-restore-key: ${{ steps.caching.outputs.BUILD_RESTORE_KEY }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Get Make recipes for caching - run: | - make deps --dry-run ${{ env.MAKE_VARS }} > make_deps.txt - make build --dry-run ${{ env.MAKE_VARS }} > make_build.txt - name: Generate cache restore keys id: caching run: | - DEPS_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_deps.txt', 'Brewfile') }}" - BUILD_RESTORE_KEY="$DEPS_RESTORE_KEY-${{ hashFiles('make_build.txt', 'configure.sh') }}" - echo "DEPS_RESTORE_KEY=$DEPS_RESTORE_KEY" >> "$GITHUB_OUTPUT" + BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', 'bundle_deps.py', 'configure.sh', 'relativize_install_names.py') }}" echo "BUILD_RESTORE_KEY=$BUILD_RESTORE_KEY" >> "$GITHUB_OUTPUT" - - name: Look up cached deps - id: cache_deps - if: inputs.use-cached - uses: actions/cache/restore@v3 - with: - path: build/*.sparsebundle - key: ignore - restore-keys: | - ${{ steps.caching.outputs.DEPS_RESTORE_KEY }} - lookup-only: true - - name: Make deps - if: steps.cache_deps.outputs.cache-matched-key == '' - run: | - make deps ${{ env.MAKE_VARS }} - - name: Save deps to cache - if: steps.cache_deps.outputs.cache-matched-key == '' - uses: actions/cache/save@v3 - with: - path: build/*.sparsebundle - key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}-${{ github.run_id }} - - build: - needs: deps - runs-on: ${{ inputs.build-os || 'macos-12' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - name: Restore cached build if available if: inputs.use-cached id: cache_build @@ -134,29 +98,17 @@ jobs: path: build/*.sparsebundle key: ignore restore-keys: | - ${{ needs.deps.outputs.build-restore-key }} - - name: Restore cached deps - if: steps.cache_build.outputs.cache-matched-key == '' - id: cache_deps - uses: actions/cache/restore@v3 - with: - path: build/*.sparsebundle - key: ignore - restore-keys: | - ${{ needs.deps.outputs.deps-restore-key }} - fail-on-cache-miss: true + ${{ steps.caching.outputs.build-restore-key }} - name: Build if: steps.cache_build.outputs.cache-matched-key == '' run: | - hdiutil attach build/*.sparsebundle - make --touch deps ${{ env.MAKE_VARS }} make build ${{ env.MAKE_VARS }} - name: Save build to cache if: steps.cache_build.outputs.cache-matched-key == '' && inputs.cache-build uses: actions/cache/save@v3 with: path: build/*.sparsebundle - key: ${{ needs.deps.outputs.build-restore-key }}-${{ github.run_id }} + key: ${{ steps.caching.outputs.build-restore-key }}-${{ github.run_id }} - name: Make app run: | hdiutil attach build/*.sparsebundle diff --git a/Makefile b/Makefile index 779c298..d94957e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Build configuration -SHELL = /bin/zsh +SHELL = /bin/bash OPENFOAM_VERSION = 2312 APP_NAME = OpenFOAM-v$(OPENFOAM_VERSION) @@ -21,13 +21,6 @@ DMG_FORMAT = UDRO APP_HOMEPAGE = https://github.com/gerlero/openfoam-app APP_VERSION = TEST_DIR = build/test-v$(OPENFOAM_VERSION) - -ifeq ($(DEPENDENCIES_KIND),standalone) -DIST_NAME = openfoam$(OPENFOAM_VERSION)-app-$(shell uname -m) -else -DIST_NAME = openfoam$(OPENFOAM_VERSION)-app-$(DEPENDENCIES_KIND)-$(shell uname -m) -endif - INSTALL_DIR = /Applications ifndef OPENFOAM_GIT_BRANCH @@ -36,28 +29,14 @@ endif VOLUME = /Volumes/$(APP_NAME) - # Build targets app: build/$(APP_NAME).app -build: $(VOLUME)/build/log.txt - hdiutil detach $(VOLUME) -deps: $(VOLUME)/Brewfile.lock.json +build: $(VOLUME)/build hdiutil detach $(VOLUME) +deps: Brewfile.lock.json fetch-source: $(OPENFOAM_TARBALL) - -ifeq ($(DEPENDENCIES_KIND),both) -zip: - $(MAKE) zip DEPENDENCIES_KIND=standalone - $(MAKE) clean-app - $(MAKE) zip DEPENDENCIES_KIND=homebrew - $(MAKE) clean-app -else -zip: build/$(DIST_NAME).zip -endif - install: $(INSTALL_DIR)/$(APP_NAME).app - # Build rules VOLUME_ID_FILE = $(VOLUME)/.vol_id @@ -73,7 +52,6 @@ APP_CONTENTS = \ build/$(APP_NAME).app/Contents/MacOS/openfoam \ build/$(APP_NAME).app/Contents/MacOS/bashrc - $(INSTALL_DIR)/$(APP_NAME).app: build/$(APP_NAME).app cp -r build/$(APP_NAME).app $(INSTALL_DIR)/ @@ -114,33 +92,16 @@ build/$(APP_NAME).app/Contents/%: Contents/% mkdir -p $(@D) cp -a $< $@ -build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: $(VOLUME)/build/log.txt build/$(APP_NAME).app/Contents/Resources/icon.icns relativize_install_names.py +build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: $(VOLUME)/.VolumeIcon.icns | $(VOLUME)/build [ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME) hdiutil attach \ build/$(APP_NAME)-build.sparsebundle \ -shadow - cd $(VOLUME) \ - && "$(CURDIR)/relativize_install_names.py" - cp build/$(APP_NAME).app/Contents/Resources/icon.icns $(VOLUME)/.VolumeIcon.icns - SetFile -c icnC $(VOLUME)/.VolumeIcon.icns - SetFile -a C $(VOLUME) uuidgen > $(VOLUME_ID_FILE) cat $(VOLUME_ID_FILE) - rm -rf $(VOLUME)/homebrew - [ ! -L $(VOLUME)/usr ] || rm $(VOLUME)/usr rm -rf $(VOLUME)/build - rm -rf -- $(VOLUME)/**/.git(N) - rm -f -- $(VOLUME)/**/.DS_Store(N) -ifeq ($(DEPENDENCIES_KIND),standalone) - rm $(VOLUME)/usr/bin/brew - rm $(VOLUME)/Brewfile - rm $(VOLUME)/Brewfile.lock.json -else ifeq ($(DEPENDENCIES_KIND),homebrew) - rm -rf $(VOLUME)/usr - ln -s $(shell brew --prefix) $(VOLUME)/usr -else - $(error Invalid value for DEPENDENCIES_KIND) -endif + rm -rf $(VOLUME)/**/.git + rm -f $(VOLUME)/**/.DS_Store rm -rf $(VOLUME)/.fseventsd mkdir -p build/$(APP_NAME).app/Contents/Resources hdiutil create \ @@ -153,15 +114,35 @@ endif hdiutil detach $(VOLUME) rm build/$(APP_NAME)-build.sparsebundle.shadow -$(VOLUME)/build/log.txt: $(VOLUME)/Brewfile.lock.json $(VOLUME)/etc/prefs.sh +$(VOLUME)/.VolumeIcon.icns: Contents/Resources/icon.icns | $(VOLUME) + cp Contents/Resources/icon.icns $(VOLUME)/.VolumeIcon.icns + SetFile -c icnC $(VOLUME)/.VolumeIcon.icns + SetFile -a C $(VOLUME) + +$(VOLUME)/build: Brewfile.lock.json relativize_install_names.py | $(VOLUME)/usr cd $(VOLUME) \ && source etc/bashrc \ && foamSystemCheck \ && ( ./Allwmake -j $(WMAKE_NJOBS) -s -q -k || true ) \ - && ./Allwmake -j $(WMAKE_NJOBS) -s -log=build/log.txt + && ./Allwmake -j $(WMAKE_NJOBS) -s \ + && relativize_install_names.py + +$(VOLUME)/usr: Brewfile.lock.json | $(VOLUME) + rm -rf $(VOLUME)/usr +ifeq ($(DEPENDENCIES_KIND),standalone) + cd $(VOLUME) \ + && HOMEBREW_BUNDLE_FILE="$(CURDIR)/Brewfile" "$(CURDIR)/bundle_deps.py" +else ifeq ($(DEPENDENCIES_KIND),homebrew) + ln -s $(shell brew --prefix) $(VOLUME)/usr + cp Brewfile $(VOLUME)/ + cp Brewfile.lock.json $(VOLUME)/ +else + $(error Invalid value for DEPENDENCIES_KIND) +endif -$(VOLUME)/etc/prefs.sh: $(OPENFOAM_TARBALL) configure.sh | $(VOLUME) - setopt extendedglob && rm -rf -- $(VOLUME)/^(usr|homebrew|Brewfile*)(N) +$(VOLUME): | build/$(APP_NAME)-build.sparsebundle $(OPENFOAM_TARBALL) configure.sh + hdiutil attach build/$(APP_NAME)-build.sparsebundle + rm -rf $(VOLUME)/* ifdef OPENFOAM_TARBALL tar -xzf $(OPENFOAM_TARBALL) --strip-components 1 -C $(VOLUME) else ifdef OPENFOAM_GIT_BRANCH @@ -172,33 +153,6 @@ else ifdef OPENFOAM_GIT_BRANCH endif cd $(VOLUME) && "$(CURDIR)/configure.sh" -$(VOLUME)/Brewfile.lock.json: $(VOLUME)/Brewfile | $(VOLUME)/usr -ifeq ($(DEPENDENCIES_KIND),standalone) - HOMEBREW_RELOCATABLE_INSTALL_NAMES=1 $(VOLUME)/usr/bin/brew bundle --file $(VOLUME)/Brewfile --cleanup --verbose - $(VOLUME)/usr/bin/brew list --versions -else ifeq ($(DEPENDENCIES_KIND),homebrew) - brew bundle --file $(VOLUME)/Brewfile --no-upgrade -else - $(error Invalid value for DEPENDENCIES_KIND) -endif - -$(VOLUME)/usr: | $(VOLUME) -ifeq ($(DEPENDENCIES_KIND),standalone) - git clone https://github.com/Homebrew/brew $(VOLUME)/homebrew - mkdir -p $(VOLUME)/usr/bin - ln -s ../../homebrew/bin/brew $(VOLUME)/usr/bin/ -else ifeq ($(DEPENDENCIES_KIND),homebrew) - ln -s $(shell brew --prefix) $(VOLUME)/usr -else - $(error Invalid value for DEPENDENCIES_KIND) -endif - -$(VOLUME)/Brewfile: Brewfile | $(VOLUME) - cp Brewfile $(VOLUME)/ - -$(VOLUME): | build/$(APP_NAME)-build.sparsebundle - hdiutil attach build/$(APP_NAME)-build.sparsebundle - build/$(APP_NAME)-build.sparsebundle: mkdir -p build hdiutil create \ @@ -215,6 +169,8 @@ $(OPENFOAM_TARBALL): | $(OPENFOAM_TARBALL).sha256 $(OPENFOAM_TARBALL).sha256: $(warning No checksum file found for $(OPENFOAM_TARBALL); will skip verification) +Brewfile.lock.json: Brewfile + brew bundle # Non-build targets and rules test: test-dmg test-openfoam test-bash test-zsh diff --git a/README.md b/README.md index 59fd2fa..49fa6c2 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ cd openfoam-app make ``` -The Xcode Command Line Tools are required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM and the necessary dependencies from source may take a while. +[Homebrew](https://brew.sh) is required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM from source may take a while. ## 📄 Legal notices diff --git a/bundle_deps.py b/bundle_deps.py new file mode 100755 index 0000000..11031c6 --- /dev/null +++ b/bundle_deps.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +""" +Copy Homebrew dependencies installed with Homebrew Bundle. +""" + +import subprocess +import shutil +import os + +from pathlib import Path + +import macho + +SRC_PREFIX = Path(subprocess.run(["brew", "--prefix"], stdout=subprocess.PIPE, check=True).stdout.decode().strip()) +DST_PREFIX = Path("usr") + +def change_lib_id(lib, *, id): + subprocess.run(["install_name_tool", "-id", id, lib], check=True) + subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True) + +def copy_installed_formula(formula): + print(f"Bundling {formula}") + + src_prefix = SRC_PREFIX / "opt" / formula.name + dst_prefix = DST_PREFIX / "opt" / formula.name + + src_cellar = SRC_PREFIX / "Cellar" / formula.name + dst_cellar = DST_PREFIX / "Cellar" / formula.name + + shutil.copytree(src_cellar, dst_cellar) + + dst_prefix.parent.mkdir(exist_ok=True) + shutil.copy(src_prefix, dst_prefix, follow_symlinks=False) + + # Replace library IDs and references to other libraries (install_names) + for file in dst_cellar.rglob("*"): + if not file.is_file(): + continue + if (file.suffix == ".dylib" or file.suffix == ".so"): + macho.change_lib_id(file, id=dst_prefix.absolute() / Path(*file.relative_to(dst_cellar).parts[1:])) + if (file.suffix == "" or file.suffix == ".bin" or file.suffix == ".dylib" or file.suffix == ".so"): + for install_name in macho.get_install_names(file): + if install_name.is_absolute() and install_name.is_relative_to(SRC_PREFIX): + if install_name.is_relative_to(SRC_PREFIX): + new_install_name = DST_PREFIX.absolute() / install_name.relative_to(SRC_PREFIX) + relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent) + macho.change_install_name(file, install_name, relative_install_name) + +def get_deps(*, recursive=True): + if not recursive: + return {Path(formula) for formula in subprocess.run(["brew", "bundle", "list"], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()} + + deps = get_deps(recursive=False) + for dep in list(deps): + recursive_deps = {Path(formula) for formula in subprocess.run(["brew", "deps", dep], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()} + deps.update(recursive_deps) + + return deps + + +for formula in get_deps(): + copy_installed_formula(formula) diff --git a/macho.py b/macho.py new file mode 100644 index 0000000..c78b2f1 --- /dev/null +++ b/macho.py @@ -0,0 +1,18 @@ +""" +Utility functions for working with macOS Mach-O files. +""" +import subprocess + +from pathlib import Path + +def change_lib_id(lib, *, id): + subprocess.run(["install_name_tool", "-id", id, lib], check=True) + subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True) + +def get_install_names(file): + otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode() + install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]] + return install_names + +def change_install_name(file, old_install_name, new_install_name): + subprocess.run(["install_name_tool", "-change", old_install_name, new_install_name, file], check=True) diff --git a/relativize_install_names.py b/relativize_install_names.py index 95a8796..9661084 100755 --- a/relativize_install_names.py +++ b/relativize_install_names.py @@ -1,21 +1,24 @@ #!/usr/bin/env python3 +""" +Replace absolute references to dylibs in OpenFOAM binaries with relative references. +""" import os import subprocess from pathlib import Path +import macho + def relativize_install_names(file, lib_dirs): - otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode() - install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]] - for install_name in install_names: + for install_name in macho.get_install_names(file): if install_name.is_absolute(): for lib_dir,new_lib_dir in lib_dirs.items(): lib_dir = lib_dir.absolute() if install_name.is_relative_to(lib_dir): new_install_name = new_lib_dir.absolute() / install_name.relative_to(lib_dir) relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent) - subprocess.run(["install_name_tool", "-change", install_name, relative_install_name, file]) + macho.change_install_name(file, install_name, relative_install_name) break # Replace references to dependencies