ci.yaml

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: CI

on:
  pull_request:
    branches:
    - development
    - '[0-9]+.[0-9]+.x'
    - 'feature/**'

  # Run CI also on push to backport release branches - we sometimes push code there by cherry-picking, meaning it
  # doesn't go through CI (no PR)
  # For master all code is coming from development where it's coming from PRs (meaning it's passing CI) but there's the
  # rare scenario where the combination of merged PRs won't pass CI
  push:
    branches:
    - master
    - '[0-9]+.[0-9]+.x'

env:
  NAMESPACE: mlrun-integ-test

jobs:
  lint:
    name: Lint code (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # 3.9 is the current >= 1.3.0 python version
        python-version: [3.9]
    steps:
    - uses: actions/checkout@v3
    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip~=22.3.0
        pip install -r dev-requirements.txt
    - name: Lint
      run: make lint

  go-lint:
    name: Lint code (GO)
    runs-on: ubuntu-latest
    steps:
      - name: Dump runner context
        run: echo "$RUNNER_CONTEXT"
        env:
          RUNNER_CONTEXT: ${{ toJson(runner) }}
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v3
        with:
          go-version-file: "go/go.mod"
          cache-dependency-path: "go/go.sum"
          cache: true
      - name: Install protobuf compiler
        run: |
          sudo apt-get update && sudo apt install -y protobuf-compiler
          
          # version is coupled to `go/cmd/schemas_compiler/docker/Dockerfile`
          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28
          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2
      - name: Lint
        run: make lint-go

  tests:
    name: Run Dockerized Tests
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    # TODO: re-use the next 2 actions instead of duplicating
    - name: Extract git branch
      id: git_info
      run: |
        echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
    - name: Resolve docker cache tag
      id: docker_cache
      run: |
        export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
        export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
        echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
    - name: Run Dockerized tests
      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-dockerized

  integration-tests:
    name: Run Dockerized Integration Tests
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    # TODO: re-use the next 2 actions instead of duplicating
    - name: Extract git branch
      id: git_info
      run: |
        echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
    - name: Resolve docker cache tag
      id: docker_cache
      run: |
        export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
        export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
        echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
    - name: Run Dockerized tests
      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-integration-dockerized

  go-unit-tests:
    name: Run GO Unit Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Run GO tests
        run: |
          make test-go-unit-dockerized

  go-integration-tests:
    name: Run GO Integration Tests in Kubernetes Environment
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      # since github-actions gives us 14G only, and fills it up with some garbage
      - name: Freeing up disk space
        run: |
          "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
      - uses: manusa/actions-setup-minikube@v2.7.2
        with:
          minikube version: "v1.28.0"
          kubernetes version: "v1.25.3"
          driver: docker
          github token: ${{ github.token }}
      - name: Prepare minikube
        run: |
          minikube kubectl -- config view --flatten > kubeconfig_flatten
          echo "MLRUN_TEST_KUBECONFIG=$(pwd)/kubeconfig_flatten" >> $GITHUB_ENV
      - name: Run GO tests
        run: |
          make test-go-integration-dockerized
      - name: Output some logs in case of failure
        if: ${{ failure() }}
        # add set -x to print commands before executing to make logs reading easier
        run: |
          set -x
          minikube ip
          minikube logs
          minikube kubectl -- --namespace ${NAMESPACE} get events
          minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1
          minikube kubectl -- --namespace ${NAMESPACE} get pods
          minikube kubectl -- --namespace ${NAMESPACE} get pods -o yaml
          minikube kubectl -- --namespace ${NAMESPACE} describe pods
          set +x

  migrations-tests:
    name: Run Dockerized Migrations Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      # TODO: re-use the next 2 actions instead of duplicating
      - name: Extract git branch
        id: git_info
        run: |
          echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
      - name: Resolve docker cache tag
        id: docker_cache
        run: |
          export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
          export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
          echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
      - name: Run Dockerized DB Migration tests
        run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-migrations-dockerized

  package-tests:
    name: Run package tests (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # 3.9 is the current >= 1.3.0 python version
        # 3.7 is for clients with the previous supported python version
        python-version: [3.7, 3.9]
    steps:
    - uses: actions/checkout@v3
    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install automation scripts dependencies and add mlrun to dev packages
      run: pip install -r automation/requirements.txt && pip install -e .
    - name: Test package
      run: MLRUN_PYTHON_VERSION=${{ matrix.python-version }} make test-package

  docs:
    name: Build Project Documentation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      # TODO: re-use the next 2 actions instead of duplicating
      - name: Extract git branch
        id: git_info
        run: |
          echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
      - name: Resolve docker cache tag
        id: docker_cache
        run: |
          export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
          export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
          echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
      - name: Generate HTML docs
        run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make html-docs-dockerized
      - name: Upload generated docs
        uses: actions/upload-artifact@v3
        with:
          name: docs
          path: docs/_build/html

  backward-compatibility-tests:
    name: Run Backward Compatibility Tests
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    env:
      MLRUN_BC_TESTS_HOME_PATH: /home/runner/work/mlrun/mlrun
      MLRUN_BC_TESTS_BASE_CODE_PATH: /home/runner/work/mlrun/mlrun/base/mlrun
      MLRUN_BC_TESTS_HEAD_CODE_PATH: /home/runner/work/mlrun/mlrun/head/mlrun
      MLRUN_DOCKER_REGISTRY: ghcr.io/
    steps:
      - id: resolve_base_branch
        run: |
            echo "branch=$(echo ${GITHUB_BASE_REF#refs/heads/})" >> $GITHUB_OUTPUT
      - name: Checkout PR Base (target) Branch
        uses: actions/checkout@v3
        with:
          ref: ${{ steps.resolve_base_branch.outputs.branch }}
          path: base/mlrun
      - name: Checkout Merge Commit (requested branch merged with the target branch)
        uses: actions/checkout@v3
        with:
          path: head/mlrun
      - name: Resolve docker cache tag
        id: docker_cache
        run: |
          export version_suffix=$(echo "${{ steps.git_info.outputs.branch }}" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.')
          export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi)
          echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
      - name: Run Backward Compatibility Tests
        run: |
          cd head/mlrun
          MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-backward-compatibility-dockerized

  check-copyright:
    name: Check Copyright Existence
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: denoland/setup-deno@v1
        with:
          deno-version: v1.x
      - name: Check
        run: |
          deno run --allow-read https://deno.land/x/license_checker@v3.2.3/main.ts
      - name: Instructions For Adding Copyright
        if: ${{ failure() }}
        run: |
          echo "
          Please add the following lines to the top of your code:
          
          # Copyright 2023 Iguazio
          #
          # Licensed under the Apache License, Version 2.0 (the "License");
          # you may not use this file except in compliance with the License.
          # You may obtain a copy of the License at
          #
          #   http://www.apache.org/licenses/LICENSE-2.0
          #
          # Unless required by applicable law or agreed to in writing, software
          # distributed under the License is distributed on an "AS IS" BASIS,
          # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
          # See the License for the specific language governing permissions and
          # limitations under the License.
          "

  verify-compatible-requirements-one-after-one:
    name: Verify compatible requirements (one after one)
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # 3.9 is the current >= 1.3.0 python version
        # 3.7 is for clients with the previous supported python version
        python-version: [ 3.7, 3.9 ]
    steps:
    - uses: actions/checkout@v3
    - name: Set up python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install requirements ${{ matrix.python-version }}
      run: make install-requirements
    - name: Install Complete Requirements ${{ matrix.python-version }}
      run: make install-complete-requirements
    - name: Install All Requirements ${{ matrix.python-version }}
      run: make install-all-requirements