python.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: python-sdk

# Trigger the workflow on push or pull request
on: [push, pull_request]

env:
  KUBERNETES_VERSION: "v1.25.3"

jobs:
  check-style:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - name: Install dependencies
        run: |
          pip install --upgrade pip setuptools wheel
          pip install -r ./dev-support/style-check/python/lint-requirements.txt
          pip install -r ./dev-support/style-check/python/mypy-requirements.txt
      - name: List installed packages
        run: pip list
      - name: Check python sdk code style
        run: ./dev-support/style-check/python/lint.sh

  unit:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        python-version: ['3.7', '3.8', '3.9', '3.10']
        tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0']
        exclude:
          - python-version: '3.8'
            tf-version: '1.15.0'
          - python-version: '3.9'
            tf-version: '1.15.0'
          - python-version: '3.10'
            tf-version: '1.15.0'
          - python-version: '3.10'
            tf-version: '2.6.0'
          - python-version: '3.10'
            tf-version: '2.7.0'
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: Setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install requirements
        run: |
          pip install --upgrade pip setuptools wheel
          pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt
      - name: Install pysubmarine with tf1 and pytorch
        if: ${{ matrix.tf-version == '1.15.0' }}
        run: |
          pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch]
      - name: Install pysubmarine with tf2 and pytorch
        if: startsWith(matrix.tf-version, '2.')
        run: |
          sed -i "s/tensorflow>=2.6.0,<2.10.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py
          sed -i "s/tensorflow-estimator>=2.6.0,<2.10.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py
          pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch]
      - name: List installed packages
        run: pip list && pipdeptree
      - name: Run unit test
        working-directory: ./submarine-sdk/pysubmarine
        run: pytest -m "not e2e" --cov-report xml
      - uses: codecov/codecov-action@v3
        with:
          flags: python-unit
          files: ./submarine-sdk/pysubmarine/coverage.xml
          name: codecov-unit

  integration:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        python-version: ['3.7', '3.8', '3.9', '3.10']
        tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0']
        exclude:
          - python-version: '3.8'
            tf-version: '1.15.0'
          - python-version: '3.9'
            tf-version: '1.15.0'
          - python-version: '3.10'
            tf-version: '1.15.0'
          - python-version: '3.10'
            tf-version: '2.6.0'
          - python-version: '3.10'
            tf-version: '2.7.0'
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 50
      - name: Setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install tf1 dependencies
        if: ${{ matrix.tf-version == '1.15.0' }}
        run: |
          pip install --upgrade pip setuptools wheel
          pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch]
          pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt
      - name: Install tf2 dependencies
        if: startsWith(matrix.tf-version, '2.')
        run: |
          pip install --upgrade pip setuptools wheel
          sed -i "s/tensorflow>=2.6.0,<2.10.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py
          sed -i "s/tensorflow-estimator>=2.6.0,<2.10.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py
          pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch]
          pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt
          pip cache purge
      - name: List installed packages
        run: pip list
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: "11"
      - name: Set up Maven 3.6.3
        uses: stCarolas/setup-maven@v4
        with:
          maven-version: 3.6.3
      - name: Check version
        run: |
          mvn --version
          java -version
      - name: Create kind cluster
        run: kind create cluster --config ./.github/config/kind-config-kind.yaml --wait 3m --image kindest/node:${KUBERNETES_VERSION}
      - name: Download Istio 1.17.1
        run: |
          curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.17.1 sh -
          cd istio-1.17.1
          echo "$PWD/bin" >> $GITHUB_PATH
      - name: Install Istio
        run: istioctl install -y --set values.global.proxy.resources.requests.cpu=10m
      - name: Show K8s cluster information
        run: |
          kubectl cluster-info
          kubectl version
          kubectl get pods -n kube-system
          echo "current-context:" $(kubectl config current-context)
          echo "environment-kubeconfig:" ${KUBECONFIG}
      # Cache maven (for submarine server)
      - uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-
      - name: Build Image locally
        run: 
          .github/scripts/build-image-locally-v3.sh
          mvn clean
      - name: Start submarine
        run: bash ./.github/scripts/start-submarine.sh
      - name: Set CLI config for e2e pytest
        run: |
          submarine config set connection.hostname localhost
          submarine config set connection.port 8080
          submarine config list
      - name: Run integration test
        working-directory: ./submarine-sdk/pysubmarine
        run: pytest -m "e2e" --cov-report xml
      - uses: codecov/codecov-action@v3
        with:
          flags: python-integration
          files: ./submarine-sdk/pysubmarine/coverage.xml
          name: codecov-integration
      - name: Failure status
        run: |
          kubectl describe nodes
          kubectl get pods
          kubectl -n default get events --sort-by='{.lastTimestamp}'
        if: ${{ failure() }}

  sonarcloud:
    if: github.repository == 'apache/submarine' && github.event_name == 'push' && github.ref == 'refs/heads/master'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: SonarCloud Scan
        uses: SonarSource/sonarcloud-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
        with:
            projectBaseDir: submarine-sdk/pysubmarine
            args: >
              -Dsonar.organization=apache
              -Dsonar.projectKey=apache_submarine_python_sdk
              -Dsonar.python.version=3.7,3.8,3.9,3.10