diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 963c85f6e11bf..73dd9b6d1a7a8 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -59,8 +59,13 @@ jobs: path: matlab-arrow-ubuntu.tar.gz macos: - name: AMD64 macOS 12 MATLAB - runs-on: macos-12 + name: {{ '${{ matrix.platform.architecture}}' }} macOS {{ '${{ matrix.platform.macos-version }}' }} MATLAB + runs-on: macos-{{ '${{ matrix.platform.macos-version }}' }} + strategy: + matrix: + platform: + - { architecture: "AMD64", macos-version: "12", architecture-suffix: "x64"} + - { architecture: "ARM64", macos-version: "14", architecture-suffix: "arm64"} steps: {{ macros.github_checkout_arrow()|indent }} - name: Install ninja-build @@ -73,27 +78,18 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - - name: Change shared library dependency name - # MATLAB's programmatic packaging interface does not properly - # include symbolic link files in the package MLTBX - this is a - # bug. As a temporary workaround, change the expected name of the - # Arrow C++ library which libarrowproxy.dylib depends on. - # For example, change libarrow.1500.dylib to libarrow.1500.0.0.dylib. + - name: Update dynamic library names run: | - pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy - SYMLINK_ARROW_LIB="$(find . -name 'libarrow.*.dylib' -type l | xargs basename)" - REGULAR_ARROW_LIB="$(echo libarrow.*.*.dylib)" - echo "SYMLINK_ARROW_LIB = ${SYMLINK_ARROW_LIB}" - echo "REGULAR_ARROW_LIB = ${REGULAR_ARROW_LIB}" - install_name_tool -change @rpath/$SYMLINK_ARROW_LIB @rpath/$REGULAR_ARROW_LIB libarrowproxy.dylib - popd + arrow/dev/tasks/matlab/rename_macos_dynamic_libraries.sh \ + arrow/matlab/install/arrow_matlab/+libmexclass/+proxy \ + {{ '${{ matrix.platform.architecture-suffix }}' }} - name: Compress into single artifact - run: tar -cvzf matlab-arrow-macos.tar.gz arrow/matlab/install/arrow_matlab + run: tar -cvzf matlab-arrow-macos-{{ '${{ matrix.platform.architecture-suffix }}' }}.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: matlab-arrow-macos.tar.gz - path: matlab-arrow-macos.tar.gz + name: matlab-arrow-macos-{{ '${{ matrix.platform.architecture-suffix }}' }}.tar.gz + path: matlab-arrow-macos-{{ '${{ matrix.platform.architecture-suffix }}' }}.tar.gz windows: name: AMD64 Windows 2022 MATLAB @@ -140,7 +136,8 @@ jobs: run: | mv artifacts-downloaded/*/*.tar.gz . tar -xzvf matlab-arrow-ubuntu.tar.gz - tar -xzvf matlab-arrow-macos.tar.gz + tar -xzvf matlab-arrow-macos-x64.tar.gz + tar -xzvf matlab-arrow-macos-arm64.tar.gz tar -xzvf matlab-arrow-windows.tar.gz - name: Copy LICENSE.txt and NOTICE.txt for packaging run: | diff --git a/dev/tasks/matlab/rename_macos_dynamic_libraries.sh b/dev/tasks/matlab/rename_macos_dynamic_libraries.sh new file mode 100755 index 0000000000000..a2a66fee0b55d --- /dev/null +++ b/dev/tasks/matlab/rename_macos_dynamic_libraries.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +# Issue Number 1: +# +# MATLAB's programmatic packaging interface does not properly package symbolic links. If the +# toolboxFolder argumented provided to the constructor of matlab.addons.toolbox.ToolboxOptions +# contains a symbolic link, the ToolboxOptions will resolve the symbolic link and include +# the symbolic link's target as one of the files to package instead of the link itself. +# +# Example: +# +# Suppose you had this folder structure +# +# $ tree /tmp/example +# /tmp/example +# |-- regular_file.txt +# |-- symbolic_link -> regular_file.txt +# +# Passing "/tmp/example" as the toolboxFolder argument to ToolboxOptions' constructor in MATLAB +# returns the following object: +# +# >> opts = matlab.addons.toolbox.ToolboxOptions("/tmp/example", "dummy-identifier"); +# >> opts.ToolboxFiles +# +# ans = +# +# "/private/tmp/example/regular_file.txt" +# +# Note that the ToolboxFiles property - the list of files to package - does not include the +# symbolic link. +# +# This is a known limitation with matlab.addons.toolbox.ToolboxOptions. +# +# Why is this a problem? +# +# On macOS, building the Arrow C++ bindings generates the following files: +# +# $ tree arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ +# . +# |-- libarrow.1700.0.0.dylib +# |-- libarrow.1700.dylib -> libarrow.1700.0.0.dylib +# |-- libarrow.dylib -> libarrow.1700.dylib +# +# When "arrow/matlab/install/arrow_matlab" is suppplied as the toolboxFolder argument +# to the constructor of ToolboxOptions, only the libarrow.1700.0.0.dylib is included as a file +# to package. This is a problem because building the MATLAB interface to Arrow creates a shared +# library called libarrowproxy.dylib, which is linked against libarrow.1700.dylib - not +# libarrow.1700.0.0.dyblib. +# +# This can be seen by calling otool with the -L flag on libarrowproxy.dylib: +# +# $ otool -L libarrowproxy.dylib | grep -E '@rpath/libarrow\.' +# @rpath/libarrow.1700.dylib +# +# To prevent a run-time linker error, we need to update the name of libarrowproxy.dylib's +# dependent shared library from @rpath/libarrow.1700.dylib to @rpath/libarrow.1700.0.0.dylib +# because only libarrow.1700.0.0.dylib is packaged. +# +# ============================================================================================== +# +# Issue Number 2: +# +# The platforms that the MATLAB Interface to Arrow supports Windows, Linux, +# Intel/AMD-based macOS and ARM-based macOS. Currently, we create one "monolithic" MLTBX file +# to package the interface that includes all shared libraries for all supported platforms. +# We do this because the File Exchange <-> GitHub Release integration does not support +# platform-specific MLTBX files. +# +# The problem with creating one MLTBX to package the interface for all platforms is that +# the names of the shared libraries built by the interface for Intel/AMD-based macOS +# and ARM-based macOS are identical.For example, building the interface generates the shared +# library libarrow.1700.0.0.dylib on both platforms. To avoid this duplicate name problem, +# we need to uniquify the names of the shared libraries generated for Intel/AMD-based +# macOS and ARM-based macOS. + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +DYLIB_DIR=${1} +ARCH=${2} + +if [ "$ARCH" == "arm64" ]; then + IS_ARM64=1 +elif [ "$ARCH" == "x64" ]; then + IS_ARM64=0 +else + echo " must be arm64 or x64" + exit 1 +fi + +pushd ${DYLIB_DIR} + +LIBARROW_DYLIB="$(find . -name 'libarrow.dylib' | xargs basename)" +LIBARROW_MAJOR_DYLIB="$(find . -name 'libarrow.*.dylib' -type l | xargs basename)" +LIBARROW_MAJOR_MINOR_PATCH_DYLIB="$(echo libarrow.*.*.dylib)" +LIBMEXCLASS_DYLIB="$(find . -name 'libmexclass.dylib' | xargs basename)" +LIBARROWPROXY_DYLIB="$(find . -name 'libarrowproxy.dylib' | xargs basename)" +if [ $IS_ARM64 -eq 1 ]; then + MEX_GATEWAY="$(find . -name 'gateway.mexmaca64' | xargs basename)" +else + MEX_GATEWAY="$(find . -name 'gateway.mexmaci64' | xargs basename)" +fi + +MAJOR_MINOR_PATCH_VERSION=${LIBARROW_MAJOR_MINOR_PATCH_DYLIB#*.} +MAJOR_MINOR_PATCH_VERSION=${MAJOR_MINOR_PATCH_VERSION%.*} + +LIBARROW_ARCH_MAJOR_MINOR_PATCH_DYLIB="libarrow_${ARCH}.${MAJOR_MINOR_PATCH_VERSION}.dylib" +LIBARROWPROXY_ARCH_DYLIB="libarrowproxy_${ARCH}.dylib" +LIBMEXCLASS_ARCH_DYLIB="libmexclass_${ARCH}.dylib" + +# Delete the symbolic links. These files are not included in the packaged MLTBX file. +rm ${LIBARROW_MAJOR_DYLIB} +rm ${LIBARROW_DYLIB} + +# Rename libarrow.*.*.*.dylib to libarrow_(arm64|x64).*.*.*.dylib (e.g. libarrow.1700.0.0.dylib -> libarrow_(arm64|x64).1700.0.0.dylib) +mv ${LIBARROW_MAJOR_MINOR_PATCH_DYLIB} ${LIBARROW_ARCH_MAJOR_MINOR_PATCH_DYLIB} + +# Rename libarrowproxy.dylib to libarrowproxy_(arm64|x64).dylib +mv ${LIBARROWPROXY_DYLIB} ${LIBARROWPROXY_ARCH_DYLIB} + +# Rename libmexclass.dylib to libmexclass_(arm64|x64).dylib +mv ${LIBMEXCLASS_DYLIB} ${LIBMEXCLASS_ARCH_DYLIB} + +# Update the identificaton names of the renamed dynamic libraries +install_name_tool -id @rpath/${LIBMEXCLASS_ARCH_DYLIB} ${LIBMEXCLASS_ARCH_DYLIB} +install_name_tool -id @rpath/${LIBARROWPROXY_ARCH_DYLIB} ${LIBARROWPROXY_ARCH_DYLIB} +install_name_tool -id @rpath/${LIBARROW_ARCH_MAJOR_MINOR_PATCH_DYLIB} ${LIBARROW_ARCH_MAJOR_MINOR_PATCH_DYLIB} + +# Change install name of dependent shared library libarrow.*.*.*.dylib to libarrow_arm64.*.*.*.dylib in libarrowproxy_(arm64|x64).dylib +install_name_tool -change @rpath/${LIBARROW_MAJOR_DYLIB} @rpath/${LIBARROW_ARCH_MAJOR_MINOR_PATCH_DYLIB} ${LIBARROWPROXY_ARCH_DYLIB} + +# Change install name of dependent shared library libmexclass.dylib to libmexclass_(arm64|x64).*.*.*.dylib libarrowproxy_(arm64|x64).dylib +install_name_tool -change @rpath/${LIBMEXCLASS_DYLIB} @rpath/${LIBMEXCLASS_ARCH_DYLIB} ${LIBARROWPROXY_ARCH_DYLIB} + +# Change install name of dependent shared library libmexclass.dylib to libmexclass_(arm64|x64).dylib in gateway.(mexmaca64|mexmaci64) +install_name_tool -change @rpath/${LIBMEXCLASS_DYLIB} @rpath/${LIBMEXCLASS_ARCH_DYLIB} ${MEX_GATEWAY} + +# Change install name of dependent shared library libarrowproxy.dylib to libarrowproxy_(arm64|x64).dylib in gateway.(mexmaca64|mexmaci64) +install_name_tool -change @rpath/${LIBARROWPROXY_DYLIB} @rpath/${LIBARROWPROXY_ARCH_DYLIB} ${MEX_GATEWAY} + +popd \ No newline at end of file