Skip to content
This repository has been archived by the owner on Nov 1, 2024. It is now read-only.

TorchArrow Nightly Build and Test #959

TorchArrow Nightly Build and Test

TorchArrow Nightly Build and Test #959

Workflow file for this run

name: TorchArrow Nightly Build and Test
on:
schedule:
- cron: "0 0 * * *"
workflow_call:
secrets:
AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID:
required: true
AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY:
required: true
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
linux-container:
runs-on: ubuntu-latest
container: prestocpp/velox-sse-velox-torcharrow:kpai-20220711
strategy:
fail-fast: false
matrix:
python-version:
- 3.7
- 3.8
- 3.9
- "3.10"
steps:
- name: Check out source repository
uses: actions/checkout@v2
with:
submodules: recursive
- name: Build the wheel
run: |
source /opt/conda/etc/profile.d/conda.sh
PYTHON_VERSION=${{ matrix.python-version }} CPU_TARGET="sse" packaging/build_wheel.sh
conda activate env${{ matrix.python-version }}
pip install auditwheel
auditwheel repair dist/*.whl -w fixed_dist --plat manylinux2014_x86_64
- name: Upload wheel to S3 Storage
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
S3_PATH: "s3://pytorch/whl/nightly/"
run: |
source /opt/conda/etc/profile.d/conda.sh
conda activate env${{ matrix.python-version }}
pip install --user awscli
set -x
if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then
echo "AWS_ACCESS_KEY_ID is not set, exiting upload"
exit 1
fi
for pkg in fixed_dist/torcharrow*.whl; do
~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read
done
# We only run this part on ubuntu wheel build for python 3.7,
# since we only need to deploy the doc once.
- name: Install TorchArrow and build docs
if: matrix.python-version == '3.7'
run: |
source /opt/conda/etc/profile.d/conda.sh
conda activate env${{ matrix.python-version }}
pip install fixed_dist/*.whl
cd ./docs
pip install -r requirements.txt --user
PATH=${PATH}:~/.local/bin
make html
cd ..
- name: Deploy Docs on Push
if: matrix.python-version == '3.7'
uses: JamesIves/[email protected]
with:
token: ${{ secrets.GITHUB_TOKEN }}
branch: gh-pages # The branch the action should deploy to.
folder: docs/build/html # The folder the action should deploy.
target-folder: main
macos-container:
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
python-version:
- 3.7
- 3.8
- 3.9
- "3.10"
steps:
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Check out source repository
uses: actions/checkout@v2
with:
submodules: recursive
- name: Build the wheel
run: |
MACOSX_DEPLOYMENT_TARGET=10.15 CPU_TARGET="sse" ./csrc/velox/velox/scripts/setup-macos.sh
pip install wheel
CPU_TARGET="sse" ./packaging/build_wheel.sh
pip install delocate
delocate-wheel dist/*.whl -w fixed_dist
- name: Upload wheel to S3 Storage
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
S3_PATH: "s3://pytorch/whl/nightly/"
run: |
pip install --user awscli
set -x
if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then
echo "AWS_ACCESS_KEY_ID is not set, exiting upload"
exit 1
fi
for pkg in fixed_dist/torcharrow*.whl; do
~/.local/bin/aws s3 cp "$pkg" "$S3_PATH" --acl public-read
done