Skip to content

Build GraphScope Wheels on macOS #260

Build GraphScope Wheels on macOS

Build GraphScope Wheels on macOS #260

name: Build GraphScope Wheels on macOS
# on: [push, pull_request]
on:
workflow_dispatch:
schedule:
# The notifications for scheduled workflows are sent to the user who
# last modified the cron syntax in the workflow file.
# Trigger the workflow at 03:00(CST) every day.
- cron: '00 19 * * *'
push:
tags:
- "v*"
concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
jobs:
build-wheels:
if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
runs-on: macos-12
strategy:
matrix:
python-version: ['3.11']
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install Dependencies
run: |
brew update || true
brew install bash coreutils
alias bash=$(brew --prefix)/bin/bash
export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
sudo mkdir /opt/graphscope
sudo chown -R $(id -u):$(id -g) /opt/graphscope
# Not sure when the first installation would fail, saying that the lock of python3.10 is taken
# However the second trial would success.
python3 -m pip install click packaging
python3 gsctl.py install-deps dev || true
python3 gsctl.py install-deps dev
echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
- name: Install JDK 11
uses: actions/setup-java@v3
with:
distribution: 'zulu' # See 'Supported distributions' for available options
java-version: '11'
- name: Reinstall glog
run: |
# Essential cause glog installed by brew is broken and produce corrupted messages
brew uninstall --ignore-dependencies glog
. ~/.graphscope_env
git clone https://github.com/google/glog.git -b v0.6.0
cd glog
cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF -DWITH_GTEST=OFF -DBUILD_TESTING=OFF
make -j
sudo make install
# make delocate-wheel happy
sudo mkdir -p /usr/local/opt/glog/lib
sudo ln -s /usr/local/lib/libglog*.dylib /usr/local/opt/glog/lib/
- name: Install latest libgrape-lite
if: false
run: |
. ~/.graphscope_env
git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
cd /tmp/libgrape-lite
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
make -j4
sudo make install
- name: Install latest vineyard
if: false
run: |
source ~/.graphscope_env
python3 -m pip install libclang
git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
cd /tmp/v6d
git submodule update --init
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_PREFIX_PATH=/usr/local \
-DUSE_EXTERNAL_ETCD_LIBS=OFF \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_VINEYARD_TESTS=OFF
make -j4
sudo make install
- name: Build Server Wheel
run: |
# source environment variable
. ~/.graphscope_env
python3 -m pip install libclang
echo ${CC}
# make sure the python's local bin is in PATH (by `pip install --user`)
export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
# change the version for nightly release
# e.g. 0.15.0 -> 0.15.0a20220808
time=$(date "+%Y%m%d")
version=$(cat ${GITHUB_WORKSPACE}/VERSION)
if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]];
then
echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
fi
python3 -m pip install numpy pandas "grpcio>=1.49" "grpcio-tools>=1.49" "mypy-protobuf>=3.4.0" wheel
# build graphscope server wheel
cd ${GITHUB_WORKSPACE}/k8s/internal
sudo -E env PATH=$PATH make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local
# package
cd ${GITHUB_WORKSPACE}
tar -zcf graphscope.tar.gz coordinator/dist/wheelhouse/*.whl
- name: Setup tmate session
if: false
uses: mxschmitt/action-tmate@v2
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
name: wheel-macos-${{ github.sha }}
path: graphscope.tar.gz
retention-days: 5
build-client-wheels:
if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
runs-on: macos-12
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install Dependencies
run: |
brew update || true
brew install bash coreutils
alias bash=$(brew --prefix)/bin/bash
sudo mkdir /opt/graphscope
sudo chown -R $(id -u):$(id -g) /opt/graphscope
python3 -m pip install click packaging
python3 gsctl.py install-deps dev || true
python3 gsctl.py install-deps dev
- name: Install latest libgrape-lite
if: false
run: |
. ~/.graphscope_env
git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
cd /tmp/libgrape-lite
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
make -j4
sudo make install
- name: Install latest vineyard
if: false
run: |
. ~/.graphscope_env
python3 -m pip install libclang
git clone --single-branch -b v0.22.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
cd /tmp/v6d
git submodule update --init
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_PREFIX_PATH=/usr/local \
-DUSE_EXTERNAL_ETCD_LIBS=OFF \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_VINEYARD_TESTS=OFF
make -j4
sudo make install
- name: Build Client Wheels
run: |
python3 -c "import sys; print(sys.version)"
. ~/.graphscope_env
# make sure the python's local bin is in PATH (by `pip install --user`)
export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
# change the version for nightly release
# 0.15.0 -> 0.15.0a20220808
time=$(date "+%Y%m%d")
version=$(cat ${GITHUB_WORKSPACE}/VERSION)
if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then
echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
fi
# build graphscope client wheel
cd ${GITHUB_WORKSPACE}/k8s/internal
make graphscope-client-py3-package GRAPHSCOPE_HOME=/usr/local
# package
cd ${GITHUB_WORKSPACE}
tar -zcf client.tar.gz python/dist/
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
name: wheel-macos-${{ github.sha }}-${{ matrix.python-version }}
path: client.tar.gz
retention-days: 5
# Action gh-action-pypi-publish not support non-linux os.
publish-wheels:
if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
runs-on: ubuntu-20.04
needs: [build-wheels, build-client-wheels]
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
steps:
- uses: actions/download-artifact@v3
with:
path: artifacts
- name: Prepare Wheels
env:
PYTHON: ${{ matrix.python-version }}
run: |
# move wheels into one folder to upload to PyPI
mkdir ${GITHUB_WORKSPACE}/upload_pypi
cd ${GITHUB_WORKSPACE}/artifacts
tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz
mv ${GITHUB_WORKSPACE}/artifacts/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/
if [ "$PYTHON" == "3.8" ]; then
tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz
mv ${GITHUB_WORKSPACE}/artifacts/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/
fi
# We do this, since failures on test.pypi aren't that bad
- name: Publish to Test PyPI
# the limit of graphscope package in test.pypi is still 100MB, which is not enough.
if: false
uses: pypa/[email protected]
with:
user: __token__
password: ${{ secrets.TEST_PYPI_PASSWORD }}
repository_url: https://test.pypi.org/legacy/
packages_dir: upload_pypi/
- name: Publish distribution to PyPI
uses: pypa/[email protected]
with:
user: __token__
password: ${{ secrets.PYPI_PASSWORD }}
packages_dir: upload_pypi/
python-test:
if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }}
runs-on: macos-12
needs: [build-wheels, build-client-wheels]
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
steps:
- uses: actions/download-artifact@v3
with:
path: artifacts
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Prepare Environment
run: |
cd ${GITHUB_WORKSPACE}/artifacts
# install graphscope-client
tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz
pushd python/dist/wheelhouse
for f in * ; do python3 -m pip install $f --user || true; done
popd
# install graphscope
tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz
pushd coordinator/dist/wheelhouse
python3 -m pip install ./*.whl --user
popd
# install tensorflow
python3 -m pip install pytest "tensorflow" "pandas" --user
- name: Run Minimum Test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
run: |
export JAVA_HOME=$(/usr/libexec/java_home -v11)
export PATH=$JAVA_HOME/bin:$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
python3 -c "import sys; print(sys.version)"
git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}
python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest
- name: Setup tmate session
if: false
uses: mxschmitt/action-tmate@v2