From c9586700a22b0953298591b2fe365c83fcfd5a2f Mon Sep 17 00:00:00 2001 From: Maksym Zhytnikov <63515947+Maxxx-zh@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:10:56 +0300 Subject: [PATCH] [FSTORE-985] Update GitHub Actions with TF tutorials (#199) * TF Tutorials * All jobs into one workflow * Update API Keys --- .github/workflows/test-python-310.yml | 31 ----------- .github/workflows/test-python-38.yml | 31 ----------- .github/workflows/test-python-39.yml | 30 ----------- .github/workflows/test_tutorials.yml | 76 +++++++++++++++++++++++++++ requirements.txt | 4 +- scripts/cleanup-tutorials.ipynb | 48 +++++++++++++++++ scripts/test-notebooks.sh | 30 ++++++++++- 7 files changed, 154 insertions(+), 96 deletions(-) delete mode 100644 .github/workflows/test-python-310.yml delete mode 100644 .github/workflows/test-python-38.yml delete mode 100644 .github/workflows/test-python-39.yml create mode 100644 .github/workflows/test_tutorials.yml diff --git a/.github/workflows/test-python-310.yml b/.github/workflows/test-python-310.yml deleted file mode 100644 index 40ab386e..00000000 --- a/.github/workflows/test-python-310.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: test-tutorials-python-310 - -on: - workflow_dispatch: - push: - branches: - - 'master' - -jobs: - test_schedule: - runs-on: ubuntu-latest - steps: - - name: checkout repo content - uses: actions/checkout@v2 - - - name: setup python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - - name: install python packages - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: execute python workflows from bash script - env: - HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY_310 }} - WEATHER_API_KEY: ${{ secrets.WEATHER_API_KEY }} - run: ./scripts/test-notebooks.sh - diff --git a/.github/workflows/test-python-38.yml b/.github/workflows/test-python-38.yml deleted file mode 100644 index d63f386e..00000000 --- a/.github/workflows/test-python-38.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: test-tutorials-python-38 - -on: - workflow_dispatch: - push: - branches: - - 'master' - -jobs: - test_schedule: - runs-on: ubuntu-latest - steps: - - name: checkout repo content - uses: actions/checkout@v2 - - - name: setup python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: install python packages - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: execute python workflows from bash script - env: - HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY_38 }} - WEATHER_API_KEY: ${{ secrets.WEATHER_API_KEY }} - run: ./scripts/test-notebooks.sh - diff --git a/.github/workflows/test-python-39.yml b/.github/workflows/test-python-39.yml deleted file mode 100644 index ca821a34..00000000 --- a/.github/workflows/test-python-39.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: test-tutorials-python-39 - -on: - workflow_dispatch: - push: - branches: - - 'master' - -jobs: - test_schedule: - runs-on: ubuntu-latest - steps: - - name: checkout repo content - uses: actions/checkout@v2 - - - name: setup python - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - - name: install python packages - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: execute python workflows from bash script - env: - HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY_39 }} - WEATHER_API_KEY: ${{ secrets.WEATHER_API_KEY }} - run: ./scripts/test-notebooks.sh \ No newline at end of file diff --git a/.github/workflows/test_tutorials.yml b/.github/workflows/test_tutorials.yml new file mode 100644 index 00000000..e1fcbf71 --- /dev/null +++ b/.github/workflows/test_tutorials.yml @@ -0,0 +1,76 @@ +name: test-tutorials-python + +on: + workflow_dispatch: + push: + branches: + - 'master' + +jobs: + test_tutorials38: + runs-on: ubuntu-latest + steps: + - name: Checkout repo content + uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Execute Python workflows from bash script + env: + HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY_38 }} + WEATHER_API_KEY: ${{ secrets.WEATHER_API_KEY }} + run: ./scripts/test-notebooks.sh + + test_tutorials39: + needs: test_tutorials38 + runs-on: ubuntu-latest + steps: + - name: Checkout repo content + uses: actions/checkout@v3 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Execute Python workflows from bash script + env: + HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY_39 }} + WEATHER_API_KEY: ${{ secrets.WEATHER_API_KEY }} + run: ./scripts/test-notebooks.sh + + test_tutorials310: + needs: test_tutorials39 + runs-on: ubuntu-latest + steps: + - name: checkout repo content + uses: actions/checkout@v3 + + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: install python packages + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: execute python workflows from bash script + env: + HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY_310 }} + WEATHER_API_KEY: ${{ secrets.WEATHER_API_KEY }} + run: ./scripts/test-notebooks.sh \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 837a6bae..ff30a46e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,14 +10,12 @@ jupyter ipykernel nbconvert scikit-learn -gradio plotly Pillow seaborn xgboost joblib==1.1.0 textblob -vaderSentiment -tweepy tensorflow +torch geopy diff --git a/scripts/cleanup-tutorials.ipynb b/scripts/cleanup-tutorials.ipynb index 4014075c..c0272338 100644 --- a/scripts/cleanup-tutorials.ipynb +++ b/scripts/cleanup-tutorials.ipynb @@ -423,6 +423,54 @@ " except:\n", " print(f\"Couldn't delete {fg} FG\")" ] + }, + { + "cell_type": "markdown", + "id": "7fa8f537", + "metadata": {}, + "source": [ + "## Cleanup TF Tutorials" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d9b676d", + "metadata": {}, + "outputs": [], + "source": [ + " \n", + "# Delete a model\n", + "models=[\"xgb_model\", \"keras_model\", \"torch_model\"]\n", + "for model in models:\n", + " try:\n", + " model = mr.get_model(f\"{model}\", version=1)\n", + " model.delete()\n", + " except:\n", + " print(f\"Couldn't delete {model} model\")\n", + "\n", + "# Delete the feature_views before the feature groups\n", + "\n", + "fvs=[\"serving_fv\"]\n", + "\n", + "for fv in fvs:\n", + " try:\n", + " feature_view = fs.get_feature_view(name=f\"{fv}\", version=1)\n", + " feature_view.delete()\n", + " except:\n", + " print(f\"Couldn't delete {fv} feature view\")\n", + "\n", + "fgs = [\n", + " \"feature_group_online\",\n", + " ]\n", + "\n", + "for fg in fgs:\n", + " try:\n", + " fg = fs.get_feature_group(name=f\"{fg}\", version=1)\n", + " fg.delete()\n", + " except:\n", + " print(f\"Couldn't delete {fg} FG\")" + ] } ], "metadata": { diff --git a/scripts/test-notebooks.sh b/scripts/test-notebooks.sh index b9570dcd..1a5d1e98 100755 --- a/scripts/test-notebooks.sh +++ b/scripts/test-notebooks.sh @@ -73,4 +73,32 @@ jupyter nbconvert --to notebook --execute nyc_taxi_fares/3_nyc_taxi_fares_traini jupyter nbconvert --to notebook --execute nyc_taxi_fares/4_nyc_taxi_fares_batch_inference.ipynb # Remove any FGs, FVs, Models, Deployments -jupyter nbconvert --to notebook --execute ../scripts/cleanup-tutorials.ipynb \ No newline at end of file +jupyter nbconvert --to notebook --execute ../scripts/cleanup-tutorials.ipynb + +# Go to transformation_functions folder +cd transformation_functions + +# Keras TF +jupyter nbconvert --to notebook --execute keras/keras_transformation_functions.ipynb + +# Remove any FGs, FVs, Models, Deployments +jupyter nbconvert --to notebook --execute ../../scripts/cleanup-tutorials.ipynb + +# PyTorch TF +jupyter nbconvert --to notebook --execute pytorch/pytorch_transformation_functions.ipynb + +# Remove any FGs, FVs, Models, Deployments +jupyter nbconvert --to notebook --execute ../../scripts/cleanup-tutorials.ipynb + +# Sklearn TF +jupyter nbconvert --to notebook --execute sklearn/sklearn_transformation_functions.ipynb + +# Remove any FGs, FVs, Models, Deployments +jupyter nbconvert --to notebook --execute ../../scripts/cleanup-tutorials.ipynb + +# Custom TF +python custom/transformations.py +jupyter nbconvert --to notebook --execute custom/custom_transformation_functions.ipynb + +# Remove any FGs, FVs, Models, Deployments +jupyter nbconvert --to notebook --execute ../../scripts/cleanup-tutorials.ipynb \ No newline at end of file