updated pre-commit hooks #235
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Train model, evaluate and test champion model, build containerized model, and deploy model | |
name: CI/CD | |
on: | |
push: | |
branches: ["main"] | |
# pull_request: | |
# branches: ["main"] | |
# Run workflow manually from the Actions tab | |
workflow_dispatch: | |
jobs: | |
train: | |
runs-on: ubuntu-latest | |
steps: | |
# Checks-out repository under $GITHUB_WORKSPACE, so the job can access it | |
# Use cache action to cache the virtual environment (https://stackoverflow.com/a/62639424) | |
- uses: actions/checkout@v3 | |
- name: Set up Python 3.10.* | |
uses: actions/setup-python@v1 | |
with: | |
python-version: 3.10.* | |
- name: Get pip cache dir | |
id: pip-cache | |
run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV | |
# Use cache action to cache the virtual environment | |
- name: Cache pip dependencies | |
uses: actions/cache@v2 | |
with: | |
path: ${{ env.PIP_CACHE_DIR }} | |
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install dependencies | |
run: | | |
make install | |
- name: Add project path to sys.path | |
run: | | |
echo "PYTHONPATH=${{github.workspace}}" >> $GITHUB_ENV | |
- name: Sort imports with isort | |
run: | | |
make isort | |
- name: Format with black | |
run: | | |
make format | |
- name: Test with pytest | |
run: | | |
make test | |
- name: Upload coverage reports to Codecov | |
uses: codecov/[email protected] | |
with: | |
token: ${{ secrets.CODECOV_TOKEN }} | |
slug: Adeemy/end-to-end-ml | |
- name: Lint with pylint | |
run: | | |
make lint | |
- name: Prepare data | |
run: | | |
make prep_data | |
- name: Setup Feast | |
run: | | |
make setup_feast | |
- name: Split data | |
run: | | |
make split_data | |
# # The train step is resource intensive and should be on a local machine | |
# # or on a remote compute cluster. When training is finished, this workflow | |
# # can be resumed to evaluate the models and test the champion model. This | |
# # step is not implemented in this project. Thus, the training is run on a | |
# # local machine at this juncture and train step is included in the CI/CD | |
# # pipeline for demonstration purposes. | |
# - name: Trigger train | |
# env: | |
# COMET_API_KEY: ${{ secrets.COMET_API_KEY }} | |
# TRAINING_SSH_KEY: ${{ secrets.TRAINING_SSH_KEY }} | |
# run: | | |
# ssh -i ${{ secrets.TRAINING_SSH_KEY }} training@my-cluster "make train" | |
# # or use a web service: | |
# # curl -X POST https://my-cluster/start-training | |
evaluate: | |
runs-on: ubuntu-latest | |
needs: train | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python 3.10.* | |
uses: actions/setup-python@v1 | |
with: | |
python-version: 3.10.* | |
- name: Get pip cache dir | |
id: pip-cache | |
run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV | |
# Use cache action to cache the virtual environment | |
- name: Cache pip dependencies | |
uses: actions/cache@v2 | |
with: | |
path: ${{ env.PIP_CACHE_DIR }} | |
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
restore-keys: | | |
${{ runner.os }}-pip- | |
- name: Install dependencies | |
run: | | |
make install | |
- name: Add project path to sys.path | |
run: | | |
echo "PYTHONPATH=${{github.workspace}}" >> $GITHUB_ENV | |
# The latest version of the champion model is pulled from model registry | |
# and evaluated in this step. If the model performance meets the deployment | |
# criteria, the model is then packaged in the build job to deployed in the | |
# production environment. | |
- name: Evaluate models | |
env: | |
COMET_API_KEY: ${{ secrets.COMET_API_KEY }} | |
run: | | |
make evaluate | |
- name: Test champion model | |
env: | |
COMET_API_KEY: ${{ secrets.COMET_API_KEY }} | |
run: | | |
make test_model | |
build: | |
runs-on: ubuntu-latest | |
needs: evaluate | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v1 | |
# Login using same GitHub Actions credentials | |
- name: Log in to GitHub container registry | |
uses: docker/[email protected] | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ github.token }} | |
# Ensures that repo name is lowercase, which is a requirement for publishing to ghcr.io | |
- name: Lowercase the repo name and username | |
run: echo "REPO=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} | |
- name: Build and push container image to registry | |
uses: docker/build-push-action@v2 | |
with: | |
push: true | |
tags: ghcr.io/${{ env.REPO }}:${{ github.sha }} | |
file: ./src/inference/Dockerfile |