Skip to content

Commit

Permalink
Update dependency bundling
Browse files Browse the repository at this point in the history
  • Loading branch information
gerlero committed Dec 28, 2023
1 parent d69942c commit 5fb83e3
Show file tree
Hide file tree
Showing 6 changed files with 124 additions and 133 deletions.
56 changes: 4 additions & 52 deletions .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,54 +78,18 @@ env:
OPENFOAM: ${{ inputs.openfoam-version || inputs.openfoam-git-branch }}

jobs:
deps:
build:
runs-on: ${{ inputs.build-os || 'macos-12' }}
env:
BUILD_OS: ${{ inputs.build-os || 'macos-12' }}
outputs:
deps-restore-key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}
build-restore-key: ${{ steps.caching.outputs.BUILD_RESTORE_KEY }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get Make recipes for caching
run: |
make deps --dry-run ${{ env.MAKE_VARS }} > make_deps.txt
make build --dry-run ${{ env.MAKE_VARS }} > make_build.txt
- name: Generate cache restore keys
id: caching
run: |
DEPS_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_deps.txt', 'Brewfile') }}"
BUILD_RESTORE_KEY="$DEPS_RESTORE_KEY-${{ hashFiles('make_build.txt', 'configure.sh') }}"
echo "DEPS_RESTORE_KEY=$DEPS_RESTORE_KEY" >> "$GITHUB_OUTPUT"
BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', 'bundle_deps.py', 'configure.sh', 'relativize_install_names.py') }}"
echo "BUILD_RESTORE_KEY=$BUILD_RESTORE_KEY" >> "$GITHUB_OUTPUT"
- name: Look up cached deps
id: cache_deps
if: inputs.use-cached
uses: actions/cache/restore@v3
with:
path: build/*.sparsebundle
key: ignore
restore-keys: |
${{ steps.caching.outputs.DEPS_RESTORE_KEY }}
lookup-only: true
- name: Make deps
if: steps.cache_deps.outputs.cache-matched-key == ''
run: |
make deps ${{ env.MAKE_VARS }}
- name: Save deps to cache
if: steps.cache_deps.outputs.cache-matched-key == ''
uses: actions/cache/save@v3
with:
path: build/*.sparsebundle
key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}-${{ github.run_id }}

build:
needs: deps
runs-on: ${{ inputs.build-os || 'macos-12' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Restore cached build if available
if: inputs.use-cached
id: cache_build
Expand All @@ -134,29 +98,17 @@ jobs:
path: build/*.sparsebundle
key: ignore
restore-keys: |
${{ needs.deps.outputs.build-restore-key }}
- name: Restore cached deps
if: steps.cache_build.outputs.cache-matched-key == ''
id: cache_deps
uses: actions/cache/restore@v3
with:
path: build/*.sparsebundle
key: ignore
restore-keys: |
${{ needs.deps.outputs.deps-restore-key }}
fail-on-cache-miss: true
${{ steps.caching.outputs.build-restore-key }}
- name: Build
if: steps.cache_build.outputs.cache-matched-key == ''
run: |
hdiutil attach build/*.sparsebundle
make --touch deps ${{ env.MAKE_VARS }}
make build ${{ env.MAKE_VARS }}
- name: Save build to cache
if: steps.cache_build.outputs.cache-matched-key == '' && inputs.cache-build
uses: actions/cache/save@v3
with:
path: build/*.sparsebundle
key: ${{ needs.deps.outputs.build-restore-key }}-${{ github.run_id }}
key: ${{ steps.caching.outputs.build-restore-key }}-${{ github.run_id }}
- name: Make app
run: |
hdiutil attach build/*.sparsebundle
Expand Down
108 changes: 32 additions & 76 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Build configuration
SHELL = /bin/zsh
SHELL = /bin/bash
OPENFOAM_VERSION = 2312
APP_NAME = OpenFOAM-v$(OPENFOAM_VERSION)

Expand All @@ -21,13 +21,6 @@ DMG_FORMAT = UDRO
APP_HOMEPAGE = https://github.com/gerlero/openfoam-app
APP_VERSION =
TEST_DIR = build/test-v$(OPENFOAM_VERSION)

ifeq ($(DEPENDENCIES_KIND),standalone)
DIST_NAME = openfoam$(OPENFOAM_VERSION)-app-$(shell uname -m)
else
DIST_NAME = openfoam$(OPENFOAM_VERSION)-app-$(DEPENDENCIES_KIND)-$(shell uname -m)
endif

INSTALL_DIR = /Applications

ifndef OPENFOAM_GIT_BRANCH
Expand All @@ -36,28 +29,14 @@ endif

VOLUME = /Volumes/$(APP_NAME)


# Build targets
app: build/$(APP_NAME).app
build: $(VOLUME)/build/log.txt
hdiutil detach $(VOLUME)
deps: $(VOLUME)/Brewfile.lock.json
build: $(VOLUME)/build
hdiutil detach $(VOLUME)
deps: Brewfile.lock.json
fetch-source: $(OPENFOAM_TARBALL)

ifeq ($(DEPENDENCIES_KIND),both)
zip:
$(MAKE) zip DEPENDENCIES_KIND=standalone
$(MAKE) clean-app
$(MAKE) zip DEPENDENCIES_KIND=homebrew
$(MAKE) clean-app
else
zip: build/$(DIST_NAME).zip
endif

install: $(INSTALL_DIR)/$(APP_NAME).app


# Build rules
VOLUME_ID_FILE = $(VOLUME)/.vol_id

Expand All @@ -73,7 +52,6 @@ APP_CONTENTS = \
build/$(APP_NAME).app/Contents/MacOS/openfoam \
build/$(APP_NAME).app/Contents/MacOS/bashrc


$(INSTALL_DIR)/$(APP_NAME).app: build/$(APP_NAME).app
cp -r build/$(APP_NAME).app $(INSTALL_DIR)/

Expand Down Expand Up @@ -114,33 +92,16 @@ build/$(APP_NAME).app/Contents/%: Contents/%
mkdir -p $(@D)
cp -a $< $@

build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: $(VOLUME)/build/log.txt build/$(APP_NAME).app/Contents/Resources/icon.icns relativize_install_names.py
build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: $(VOLUME)/.VolumeIcon.icns | $(VOLUME)/build
[ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME)
hdiutil attach \
build/$(APP_NAME)-build.sparsebundle \
-shadow
cd $(VOLUME) \
&& "$(CURDIR)/relativize_install_names.py"
cp build/$(APP_NAME).app/Contents/Resources/icon.icns $(VOLUME)/.VolumeIcon.icns
SetFile -c icnC $(VOLUME)/.VolumeIcon.icns
SetFile -a C $(VOLUME)
uuidgen > $(VOLUME_ID_FILE)
cat $(VOLUME_ID_FILE)
rm -rf $(VOLUME)/homebrew
[ ! -L $(VOLUME)/usr ] || rm $(VOLUME)/usr
rm -rf $(VOLUME)/build
rm -rf -- $(VOLUME)/**/.git(N)
rm -f -- $(VOLUME)/**/.DS_Store(N)
ifeq ($(DEPENDENCIES_KIND),standalone)
rm $(VOLUME)/usr/bin/brew
rm $(VOLUME)/Brewfile
rm $(VOLUME)/Brewfile.lock.json
else ifeq ($(DEPENDENCIES_KIND),homebrew)
rm -rf $(VOLUME)/usr
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
rm -rf $(VOLUME)/**/.git
rm -f $(VOLUME)/**/.DS_Store
rm -rf $(VOLUME)/.fseventsd
mkdir -p build/$(APP_NAME).app/Contents/Resources
hdiutil create \
Expand All @@ -153,15 +114,35 @@ endif
hdiutil detach $(VOLUME)
rm build/$(APP_NAME)-build.sparsebundle.shadow

$(VOLUME)/build/log.txt: $(VOLUME)/Brewfile.lock.json $(VOLUME)/etc/prefs.sh
$(VOLUME)/.VolumeIcon.icns: Contents/Resources/icon.icns | $(VOLUME)
cp Contents/Resources/icon.icns $(VOLUME)/.VolumeIcon.icns
SetFile -c icnC $(VOLUME)/.VolumeIcon.icns
SetFile -a C $(VOLUME)

$(VOLUME)/build: Brewfile.lock.json relativize_install_names.py | $(VOLUME)/usr
cd $(VOLUME) \
&& source etc/bashrc \
&& foamSystemCheck \
&& ( ./Allwmake -j $(WMAKE_NJOBS) -s -q -k || true ) \
&& ./Allwmake -j $(WMAKE_NJOBS) -s -log=build/log.txt
&& ./Allwmake -j $(WMAKE_NJOBS) -s \
&& relativize_install_names.py

$(VOLUME)/usr: Brewfile.lock.json | $(VOLUME)
rm -rf $(VOLUME)/usr
ifeq ($(DEPENDENCIES_KIND),standalone)
cd $(VOLUME) \
&& HOMEBREW_BUNDLE_FILE="$(CURDIR)/Brewfile" "$(CURDIR)/bundle_deps.py"
else ifeq ($(DEPENDENCIES_KIND),homebrew)
ln -s $(shell brew --prefix) $(VOLUME)/usr
cp Brewfile $(VOLUME)/
cp Brewfile.lock.json $(VOLUME)/
else
$(error Invalid value for DEPENDENCIES_KIND)
endif

$(VOLUME)/etc/prefs.sh: $(OPENFOAM_TARBALL) configure.sh | $(VOLUME)
setopt extendedglob && rm -rf -- $(VOLUME)/^(usr|homebrew|Brewfile*)(N)
$(VOLUME): | build/$(APP_NAME)-build.sparsebundle $(OPENFOAM_TARBALL) configure.sh
hdiutil attach build/$(APP_NAME)-build.sparsebundle
rm -rf $(VOLUME)/*
ifdef OPENFOAM_TARBALL
tar -xzf $(OPENFOAM_TARBALL) --strip-components 1 -C $(VOLUME)
else ifdef OPENFOAM_GIT_BRANCH
Expand All @@ -172,33 +153,6 @@ else ifdef OPENFOAM_GIT_BRANCH
endif
cd $(VOLUME) && "$(CURDIR)/configure.sh"

$(VOLUME)/Brewfile.lock.json: $(VOLUME)/Brewfile | $(VOLUME)/usr
ifeq ($(DEPENDENCIES_KIND),standalone)
HOMEBREW_RELOCATABLE_INSTALL_NAMES=1 $(VOLUME)/usr/bin/brew bundle --file $(VOLUME)/Brewfile --cleanup --verbose
$(VOLUME)/usr/bin/brew list --versions
else ifeq ($(DEPENDENCIES_KIND),homebrew)
brew bundle --file $(VOLUME)/Brewfile --no-upgrade
else
$(error Invalid value for DEPENDENCIES_KIND)
endif

$(VOLUME)/usr: | $(VOLUME)
ifeq ($(DEPENDENCIES_KIND),standalone)
git clone https://github.com/Homebrew/brew $(VOLUME)/homebrew
mkdir -p $(VOLUME)/usr/bin
ln -s ../../homebrew/bin/brew $(VOLUME)/usr/bin/
else ifeq ($(DEPENDENCIES_KIND),homebrew)
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif

$(VOLUME)/Brewfile: Brewfile | $(VOLUME)
cp Brewfile $(VOLUME)/

$(VOLUME): | build/$(APP_NAME)-build.sparsebundle
hdiutil attach build/$(APP_NAME)-build.sparsebundle

build/$(APP_NAME)-build.sparsebundle:
mkdir -p build
hdiutil create \
Expand All @@ -215,6 +169,8 @@ $(OPENFOAM_TARBALL): | $(OPENFOAM_TARBALL).sha256
$(OPENFOAM_TARBALL).sha256:
$(warning No checksum file found for $(OPENFOAM_TARBALL); will skip verification)

Brewfile.lock.json: Brewfile
brew bundle

# Non-build targets and rules
test: test-dmg test-openfoam test-bash test-zsh
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ cd openfoam-app
make
```
The Xcode Command Line Tools are required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM and the necessary dependencies from source may take a while.
[Homebrew](https://brew.sh) is required. See the available configuration variables and alternative targets for `make` in the [`Makefile`](Makefile). Note that the compilation of OpenFOAM from source may take a while.
## 📄 Legal notices
Expand Down
62 changes: 62 additions & 0 deletions bundle_deps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""
Copy Homebrew dependencies installed with Homebrew Bundle.
"""

import subprocess
import shutil
import os

from pathlib import Path

import macho

SRC_PREFIX = Path(subprocess.run(["brew", "--prefix"], stdout=subprocess.PIPE, check=True).stdout.decode().strip())
DST_PREFIX = Path("usr")

def change_lib_id(lib, *, id):
subprocess.run(["install_name_tool", "-id", id, lib], check=True)
subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True)

def copy_installed_formula(formula):
print(f"Bundling {formula}")

src_prefix = SRC_PREFIX / "opt" / formula.name
dst_prefix = DST_PREFIX / "opt" / formula.name

src_cellar = SRC_PREFIX / "Cellar" / formula.name
dst_cellar = DST_PREFIX / "Cellar" / formula.name

shutil.copytree(src_cellar, dst_cellar)

dst_prefix.parent.mkdir(exist_ok=True)
shutil.copy(src_prefix, dst_prefix, follow_symlinks=False)

# Replace library IDs and references to other libraries (install_names)
for file in dst_cellar.rglob("*"):
if not file.is_file():
continue
if (file.suffix == ".dylib" or file.suffix == ".so"):
macho.change_lib_id(file, id=dst_prefix.absolute() / Path(*file.relative_to(dst_cellar).parts[1:]))
if (file.suffix == "" or file.suffix == ".bin" or file.suffix == ".dylib" or file.suffix == ".so"):
for install_name in macho.get_install_names(file):
if install_name.is_absolute() and install_name.is_relative_to(SRC_PREFIX):
if install_name.is_relative_to(SRC_PREFIX):
new_install_name = DST_PREFIX.absolute() / install_name.relative_to(SRC_PREFIX)
relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent)
macho.change_install_name(file, install_name, relative_install_name)

def get_deps(*, recursive=True):
if not recursive:
return {Path(formula) for formula in subprocess.run(["brew", "bundle", "list"], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()}

deps = get_deps(recursive=False)
for dep in list(deps):
recursive_deps = {Path(formula) for formula in subprocess.run(["brew", "deps", dep], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()}
deps.update(recursive_deps)

return deps


for formula in get_deps():
copy_installed_formula(formula)
18 changes: 18 additions & 0 deletions macho.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
"""
Utility functions for working with macOS Mach-O files.
"""
import subprocess

from pathlib import Path

def change_lib_id(lib, *, id):
subprocess.run(["install_name_tool", "-id", id, lib], check=True)
subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True)

def get_install_names(file):
otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode()
install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]]
return install_names

def change_install_name(file, old_install_name, new_install_name):
subprocess.run(["install_name_tool", "-change", old_install_name, new_install_name, file], check=True)
11 changes: 7 additions & 4 deletions relativize_install_names.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
#!/usr/bin/env python3
"""
Replace absolute references to dylibs in OpenFOAM binaries with relative references.
"""

import os
import subprocess

from pathlib import Path

import macho

def relativize_install_names(file, lib_dirs):
otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode()
install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]]
for install_name in install_names:
for install_name in macho.get_install_names(file):
if install_name.is_absolute():
for lib_dir,new_lib_dir in lib_dirs.items():
lib_dir = lib_dir.absolute()
if install_name.is_relative_to(lib_dir):
new_install_name = new_lib_dir.absolute() / install_name.relative_to(lib_dir)
relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent)
subprocess.run(["install_name_tool", "-change", install_name, relative_install_name, file])
macho.change_install_name(file, install_name, relative_install_name)
break

# Replace references to dependencies
Expand Down

0 comments on commit 5fb83e3

Please sign in to comment.