setup: Vendor some old source-only dependencies #11417
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: default | |
| on: | |
| push: | |
| pull_request: | |
| types: [labeled, unlabeled, opened, synchronize, reopened] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Calculate the fetch depth | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then | |
| echo "GIT_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" | |
| else | |
| echo "GIT_FETCH_DEPTH=2" >> "${GITHUB_ENV}" | |
| fi | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: ${{ env.GIT_FETCH_DEPTH }} | |
| - name: Extract Python version from pants.toml | |
| run: | | |
| PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml) | |
| echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV | |
| echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV | |
| - name: Set up Python as Runtime | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PROJECT_PYTHON_VERSION }} | |
| - name: Bootstrap Pants | |
| uses: pantsbuild/actions/init-pants@v5-scie-pants | |
| # See: github.com/pantsbuild/actions/tree/main/init-pants/ | |
| # ref) https://github.com/pantsbuild/example-python/blob/main/.github/workflows/pants.yaml#L30-L49 | |
| with: | |
| gha-cache-key: pants-cache-main-1-lint-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }} | |
| cache-lmdb-store: 'true' | |
| - name: Check BUILD files | |
| run: pants tailor --check update-build-files --check '::' | |
| - name: Check forbidden cross imports | |
| run: ./scripts/check-cross-imports.sh | |
| - name: Lint | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" -a -n "$GITHUB_HEAD_REF" ]; then | |
| echo "(skipping matchers for pull request from local branches)" | |
| else | |
| echo "::add-matcher::.github/workflows/flake8-matcher.json" | |
| fi | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then | |
| [ -n "$GITHUB_BASE_REF" ] && BASE_REF_SHORT="${GITHUB_BASE_REF}" || BASE_REF_SHORT="main" | |
| BASE_REF="origin/${BASE_REF_SHORT}" | |
| git remote set-branches origin "$BASE_REF_SHORT" | |
| BASE_COMMIT=$(git rev-list --first-parent --max-parents=0 --max-count=1 HEAD) | |
| BASE_TIMESTAMP=$(git log --format=%ct "${BASE_COMMIT}") | |
| git fetch --no-tags --shallow-since "${BASE_TIMESTAMP}" origin "${BASE_REF_SHORT}" | |
| else | |
| BASE_REF="HEAD~1" | |
| fi | |
| pants lint --changed-since=$BASE_REF --changed-dependents=transitive | |
| - name: Upload pants log | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pants.lint.log | |
| path: .pants.d/pants.log | |
| if: always() # We want the log even on failures. | |
| typecheck: | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} | |
| runs-on: ubuntu-latest-8-cores | |
| steps: | |
| - name: Calculate the fetch depth | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then | |
| echo "GIT_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" | |
| else | |
| echo "GIT_FETCH_DEPTH=2" >> "${GITHUB_ENV}" | |
| fi | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: ${{ env.GIT_FETCH_DEPTH }} | |
| - name: Extract Python version from pants.toml | |
| run: | | |
| PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml) | |
| echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV | |
| echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV | |
| - name: Set up Python as Runtime | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PROJECT_PYTHON_VERSION }} | |
| - name: Bootstrap Pants | |
| uses: pantsbuild/actions/init-pants@v5-scie-pants | |
| with: | |
| gha-cache-key: pants-cache-main-1-typecheck-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }} | |
| cache-lmdb-store: 'true' | |
| - name: Typecheck | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" -a -n "$GITHUB_HEAD_REF" ]; then | |
| echo "(skipping matchers for pull request from local branches)" | |
| else | |
| echo "::add-matcher::.github/workflows/mypy-matcher.json" | |
| fi | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then | |
| [ -n "$GITHUB_BASE_REF" ] && BASE_REF_SHORT="${GITHUB_BASE_REF}" || BASE_REF_SHORT="main" | |
| BASE_REF="origin/${BASE_REF_SHORT}" | |
| git remote set-branches origin "$BASE_REF_SHORT" | |
| BASE_COMMIT=$(git rev-list --first-parent --max-parents=0 --max-count=1 HEAD) | |
| BASE_TIMESTAMP=$(git log --format=%ct "${BASE_COMMIT}") | |
| git fetch --no-tags --shallow-since "${BASE_TIMESTAMP}" origin "${BASE_REF_SHORT}" | |
| else | |
| BASE_REF="HEAD~1" | |
| fi | |
| pants check --changed-since=$BASE_REF --changed-dependents=transitive | |
| - name: Upload pants log | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pants.check.log | |
| path: .pants.d/pants.log | |
| if: always() # We want the log even on failures. | |
| test: | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} | |
| runs-on: ubuntu-latest-8-cores | |
| steps: | |
| - name: Calculate the fetch depth | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then | |
| echo "GIT_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" | |
| else | |
| echo "GIT_FETCH_DEPTH=2" >> "${GITHUB_ENV}" | |
| fi | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: ${{ env.GIT_FETCH_DEPTH }} | |
| - name: Create LFS file hash list | |
| run: git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-id | |
| - name: Restore LFS cache | |
| uses: actions/cache@v3 | |
| id: lfs-cache | |
| with: | |
| path: .git/lfs | |
| key: lfs-${{ hashFiles('.lfs-assets-id') }} | |
| - name: Git LFS Pull | |
| run: git lfs pull | |
| - name: Extract Python version from pants.toml | |
| run: | | |
| PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml) | |
| echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV | |
| echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV | |
| - name: Set up Python as Runtime | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PROJECT_PYTHON_VERSION }} | |
| - name: Bootstrap Pants | |
| uses: pantsbuild/actions/init-pants@v5-scie-pants | |
| with: | |
| gha-cache-key: pants-cache-main-1-test-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }} | |
| cache-lmdb-store: 'true' | |
| - name: Test | |
| timeout-minutes: 15 | |
| run: | | |
| # configure redis sentinel cluster hostnames for testing | |
| echo "127.0.0.1 node01" | sudo tee -a /etc/hosts | |
| echo "127.0.0.1 node02" | sudo tee -a /etc/hosts | |
| echo "127.0.0.1 node03" | sudo tee -a /etc/hosts | |
| if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then | |
| [ -n "$GITHUB_BASE_REF" ] && BASE_REF_SHORT="${GITHUB_BASE_REF}" || BASE_REF_SHORT="main" | |
| BASE_REF="origin/${BASE_REF_SHORT}" | |
| git remote set-branches origin "$BASE_REF_SHORT" | |
| BASE_COMMIT=$(git rev-list --first-parent --max-parents=0 --max-count=1 HEAD) | |
| BASE_TIMESTAMP=$(git log --format=%ct "${BASE_COMMIT}") | |
| git fetch --no-tags --shallow-since "${BASE_TIMESTAMP}" origin "${BASE_REF_SHORT}" | |
| else | |
| BASE_REF="HEAD~1" | |
| fi | |
| pants test --changed-since=$BASE_REF --changed-dependents=transitive -- -m 'not integration' -v | |
| - name: Upload pants log | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pants.test.log | |
| path: .pants.d/pants.log | |
| if: always() # We want the log even on failures. | |
| build-wheels: | |
| needs: [lint, typecheck, test] | |
| if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| environment: deploy-to-pypi | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Fetch remote tags | |
| run: git fetch origin 'refs/tags/*:refs/tags/*' -f | |
| - name: Create LFS file hash list | |
| run: git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-id | |
| - name: Restore LFS cache | |
| uses: actions/cache@v3 | |
| id: lfs-cache | |
| with: | |
| path: .git/lfs | |
| key: lfs-${{ hashFiles('.lfs-assets-id') }} | |
| - name: Git LFS Pull | |
| run: git lfs pull | |
| - name: Extract Python version from pants.toml | |
| run: | | |
| PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml) | |
| echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV | |
| echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV | |
| - name: Prepare the pip cache directory | |
| run: mkdir -p ~/.cache/pip | |
| - name: Set up Python as Runtime | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PROJECT_PYTHON_VERSION }} | |
| - name: Bootstrap Pants | |
| uses: pantsbuild/actions/init-pants@v5-scie-pants | |
| with: | |
| gha-cache-key: pants-cache-main-1-deploy-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }} | |
| named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }} | |
| cache-lmdb-store: 'true' | |
| - name: Install local dependencies for packaging | |
| run: | | |
| pip install -U 'twine~=4.0' 'packaging>=21.3' | |
| - name: Build packages | |
| run: | | |
| # Normalize the package version | |
| PKGVER=$(python -c "import packaging.version,pathlib; print(str(packaging.version.Version(pathlib.Path('VERSION').read_text())))") | |
| echo "PKGVER=$PKGVER" >> $GITHUB_ENV | |
| # Build non-platform-specific wheels | |
| pants --platform-specific-resources-target=linux_x86_64 --tag="wheel" --tag="-platform-specific" package '::' | |
| # Build x86_64 wheels | |
| MANYLINUX_PTAG=manylinux2014_x86_64 | |
| MACOS_PTAG=macosx_11_0_x86_64 | |
| pants --platform-specific-resources-target=linux_x86_64 --tag="wheel" --tag="+platform-specific" package '::' | |
| for pkgname in "kernel_binary"; do | |
| mv "dist/backend.ai_${pkgname}-${PKGVER}-py3-none-any.whl" \ | |
| "dist/backend.ai_${pkgname}-${PKGVER}-py3-none-${MANYLINUX_PTAG}.${MACOS_PTAG}.whl" | |
| done | |
| # Build arm64 wheels | |
| MANYLINUX_PTAG=manylinux2014_aarch64 | |
| MACOS_PTAG=macosx_11_0_arm64 | |
| pants --platform-specific-resources-target=linux_arm64 --tag="wheel" --tag="+platform-specific" package '::' | |
| for pkgname in "kernel_binary"; do | |
| mv "dist/backend.ai_${pkgname}-${PKGVER}-py3-none-any.whl" \ | |
| "dist/backend.ai_${pkgname}-${PKGVER}-py3-none-${MANYLINUX_PTAG}.${MACOS_PTAG}.whl" | |
| done | |
| ls -lh dist | |
| - name: Publish to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| # We don't use `pants publish ::` because we manually rename the | |
| # wheels after buildling them to add arch-specific tags. | |
| run: | | |
| twine upload dist/*.whl dist/*.tar.gz | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist/*.whl | |
| - name: Extract the release changelog | |
| run: | | |
| python ./scripts/extract-release-changelog.py | |
| python ./scripts/determine-release-type.py | |
| - name: Release to GitHub | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| body_path: "CHANGELOG_RELEASE.md" | |
| prerelease: ${{ env.IS_PRERELEASE }} | |
| files: | | |
| dist/*.whl | |
| dist/*.tar.gz | |
| - name: Upload pants log | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pants.deploy.log | |
| path: .pants.d/pants.log | |
| if: always() # We want the log even on failures. | |
| build-conda-pack-for-windows: | |
| needs: [build-wheels] | |
| if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') | |
| runs-on: windows-latest | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Create LFS file hash list | |
| run: git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-id | |
| - name: Restore LFS cache | |
| uses: actions/cache@v3 | |
| id: lfs-cache | |
| with: | |
| path: .git/lfs | |
| key: lfs-${{ hashFiles('.lfs-assets-id') }} | |
| - name: Git LFS Pull | |
| run: git lfs pull | |
| - name: Extract Python version from pants.toml | |
| shell: bash | |
| run: | | |
| export LANG=C.UTF-8 | |
| PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml) | |
| echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV | |
| echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PROJECT_PYTHON_VERSION }} | |
| cache: pip | |
| - name: Install local dependencies for packaging | |
| run: | | |
| pip install -U 'packaging>=21.3' | |
| - name: Normalize the package version | |
| shell: bash | |
| run: | | |
| PKGVER=$(python -c "import packaging.version,pathlib; print(str(packaging.version.Version(pathlib.Path('VERSION').read_text())))") | |
| echo "PKGVER=$PKGVER" >> $GITHUB_ENV | |
| - name: Install conda-pack | |
| uses: s-weigand/setup-conda@v1 | |
| with: | |
| activate-conda: false | |
| - name: Download wheels | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist | |
| - name: Create conda environment | |
| # FIXME: Let's think about resolving dependency of backend.ai-client package programmatically, instead of hardcoding it. | |
| run: | | |
| pip install conda-pack | |
| conda create -n backend.ai-client python=${{ env.PROJECT_PYTHON_VERSION }} | |
| conda activate backend.ai-client | |
| pip install dist/backend.ai_client-${{ env.PKGVER }}-py3-none-any.whl dist/backend.ai_cli-${{ env.PKGVER }}-py3-none-any.whl dist/backend.ai_common-${{ env.PKGVER }}-py3-none-any.whl dist/backend.ai_plugin-${{ env.PKGVER }}-py3-none-any.whl | |
| conda-pack -o backend.ai-client-${{ github.ref_name }}-windows-conda.zip | |
| - name: Upload conda-pack to GitHub release | |
| run: | | |
| gh release upload ${{ github.ref_name }} backend.ai-client-${{ github.ref_name }}-windows-conda.zip | |
| build-client-image: | |
| needs: [build-wheels] | |
| runs-on: ubuntu-latest | |
| environment: deploy-to-docker-hub | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Extract Python version from pants.toml | |
| shell: bash | |
| run: | | |
| export LANG=C.UTF-8 | |
| PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml) | |
| echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV | |
| echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV | |
| - name: Build and push | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| file: ./backend.ai-client.Dockerfile | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| lablup/backend.ai-client:${{ github.ref_name }} | |
| lablup/backend.ai-client:latest | |
| build-args: | | |
| PYTHON_VERSION=${{ env.PROJECT_PYTHON_VERSION }} | |
| PKGVER=${{ github.ref_name }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max |