Skip to content

Allow csv files (#1763) #3967

Allow csv files (#1763)

Allow csv files (#1763) #3967

Workflow file for this run

name: Master
on:
push:
branches:
- master
paths-ignore:
- "docs/**"
pull_request:
paths-ignore:
- "docs/**"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
lint:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.11" ]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v2
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r dev-requirements.txt
- name: Pip info
run: python -m pip list
- name: Lint
run: pre-commit run --all --show-diff-on-failure
# This is the build system for the new example directory structure
list_examples:
runs-on: ubuntu-latest
name: "Create a list of example packages"
steps:
- uses: actions/checkout@v2
- name: "Provide the list"
id: create-example-list
run: |
# TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc)
echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \
| grep -v -e 'testing' -e 'examples' \
| grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'mnist_classifier' \
| sort \
| jq --raw-input . \
| jq --slurp . \
| jq -c .)" >> "$GITHUB_OUTPUT"
outputs:
packages: "${{ steps.create-example-list.outputs.PACKAGES }}"
serialize:
needs: [list_examples]
runs-on: "ubuntu-latest"
env:
FLYTE_SDK_RICH_TRACEBACKS: "0"
strategy:
fail-fast: false
matrix:
example: "${{ fromJson(needs.list_examples.outputs.packages) }}"
steps:
- uses: actions/checkout@v4
- uses: insightsengineering/disk-space-reclaimer@v1
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
working-directory: examples/${{ matrix.example }}
run: |
pip install uv
uv venv $GITHUB_WORKSPACE/.venv
source $GITHUB_WORKSPACE/.venv/bin/activate
if [ -f requirements.in ]; then uv pip install -r requirements.in; fi
uv pip install "flytekit>=1.12.2" "numpy<2.0.0"
pip freeze
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
if: ${{ github.event_name != 'pull_request' }}
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ secrets.FLYTE_BOT_USERNAME }}
password: ${{ secrets.FLYTE_BOT_PAT }}
- name: Check if dockerfile exists
working-directory: examples/${{ matrix.example }}
id: dockerfile
run: |
if [ -f Dockerfile ]
then
echo "exist=true" >> "$GITHUB_OUTPUT"
else
echo "exist=false" >> "$GITHUB_OUTPUT"
fi
- name: Build and push default image
if: ${{ steps.dockerfile.outputs.exist == 'true' }}
uses: docker/build-push-action@v5
with:
context: examples/${{ matrix.example }}
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }},ghcr.io/flyteorg/flytecookbook:latest
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Pyflyte package
working-directory: examples/${{ matrix.example }}
run: |
export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }}
default_image=ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }}
source $GITHUB_WORKSPACE/.venv/bin/activate
pyflyte \
--pkgs ${{ matrix.example }} package \
--image $default_image \
--image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
--image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
--output flyte-package.tgz \
--force
tar -xvf flyte-package.tgz
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: snacks-examples-${{ matrix.example }}
path: examples/${{ matrix.example }}/**/*.pb
# Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them.
register:
name: Register example to sandbox
runs-on: ubuntu-latest
needs: [ "serialize" ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: "0"
- uses: unionai/[email protected]
- name: setup download artifact dir
run: |
mkdir download-artifact
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: ./download-artifact/
- name: setup sandbox
run: |
flytectl demo start
flytectl config init
- name: Register examples
run: |
flytectl register files ./download-artifact/**/* -p flytesnacks -d development --version latest
bump_version:
name: Bump Version
if: ${{ github.event_name != 'pull_request' }}
needs: [ serialize ] # Only to ensure it can successfully build
uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
secrets:
FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
prerelease:
name: Create Prerelease
needs: [ bump_version ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: "0"
- name: Create Pre release
id: prerelease
run: |
RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \
--header 'Accept: application/vnd.github.v3+json' \
--header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
--data-raw '{
"tag_name": "'${{ needs.bump-version.outputs.version }}'",
"prerelease": true
}' | jq -r '.id')
echo ::set-output name=release_id::$RELEASE_ID
outputs:
release_id: ${{ steps.prerelease.outputs.release_id }}
# Download artifacts again and push them to the release only if this is not a pull request
release_workflow:
name: Publish artifacts to github release
runs-on: ubuntu-latest
needs: [ prerelease ]
strategy:
matrix:
python-version: [ "3.11" ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: "0"
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: setup download artifact dir
run: |
mkdir download-artifact
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: ./download-artifact/
- name: Package Examples
run: |
mkdir -p release-snacks
cd download-artifact
for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait
cd .. && sudo rm -rf download-artifact/
cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json
cp flyte_tests.txt release-snacks/flyte_tests.txt
- name: Release test manifest
uses: goreleaser/goreleaser-action@v5
with:
version: latest
args: release --rm-dist
env:
GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }}
make_release:
name: Mark github pre-release as Release
needs: [ release_workflow ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: "0"
- name: Update Release
id: release
run: |
curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \
--header 'Accept: application/vnd.github.v3+json' \
--header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \
--data-raw '{
"tag_name": "'${{ needs.bump-version.outputs.version }}'",
"prerelease": false
}'
e2e-tests:
runs-on: ubuntu-latest
env:
FLYTESNACKS_PRIORITIES: "P0"
FLYTESNACKS_VERSION: ""
timeout-minutes: 30
steps:
- uses: insightsengineering/disk-space-reclaimer@v1
- name: Set latest Flytesnacks release
if: ${{ env.FLYTESNACKS_VERSION == '' }}
run: |
FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- uses: unionai/[email protected]
- name: Setup sandbox
run: |
mkdir -p ~/.flyte/sandbox
cat << EOF > ~/.flyte/sandbox/config.yaml
task_resources:
defaults:
cpu: "0"
memory: "0"
limits:
cpu: "0"
memory: "0"
EOF
flytectl demo start --imagePullPolicy Never
- name: Install Python dependencies
run: |
pip install uv
uv venv
source .venv/bin/activate
uv pip install "flytekit>=1.12.2" flytekitplugins-deck-standard torch tabulate pyarrow pandas
pip freeze
- name: Checkout flytesnacks
uses: actions/checkout@v3
with:
repository: flyteorg/flytesnacks
path: flytesnacks
- name: Verify existence of the tests
run: |
source .venv/bin/activate
python flyte_tests_validate.py
- name: Register specific tests
run: |
source .venv/bin/activate
export FLYTE_PUSH_IMAGE_SPEC=${{ github.event_name != 'pull_request' }}
while read -r line;
do
pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
register \
--project flytesnacks \
--domain development \
--image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
--version ${{ env.FLYTESNACKS_VERSION }} \
flytesnacks/$line;
done < flyte_tests.txt
- name: Setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3