system-tests-opensource.yml

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: System Tests Open Source

on:
  push:
    branches:
      - '.+-system-tests'

  schedule:

    # * is a special character in YAML so you have to quote this string
    # Run the system tests every 3 hours
    - cron:  '0 */3 * * *'

  workflow_dispatch:
    inputs:
      docker_registry:
        description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
        required: true
        default: 'ghcr.io/'
      docker_repo:
        description: 'Docker repo to pull images from (default: mlrun)'
        required: true
        default: 'mlrun'
      clean_resources_in_teardown:
        description: 'Clean resources created by test (like project) in each test teardown (default: true - perform clean)'
        required: true
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'
      debug_enabled:
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: 'false'
        type: choice
        options:
          - 'true'
          - 'false'

env:
  NAMESPACE: mlrun
  MLRUN_API_NODE_PORT: 30070

jobs:
  run-system-tests-opensource-ci:
    name: Run System Tests Open Source
    runs-on: ubuntu-latest

    # let's not run this on every fork, change to your fork when developing
    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'

    steps:
    - uses: actions/checkout@v3
    # since github-actions gives us 14G only, and fills it up with some garbage
    - name: Freeing up disk space
      run: |
        "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
    - name: Set up python
      uses: actions/setup-python@v4
      with:
        python-version: 3.9
        cache: pip
    - name: Install automation scripts dependencies and add mlrun to dev packages
      run: |
        pip install \
          -r automation/requirements.txt \
          -r dockerfiles/test-system/requirements.txt \
          -r dockerfiles/mlrun-api/requirements.txt \
          -r dev-requirements.txt \
          -r extras-requirements.txt \
        && pip install -e .
        sudo apt-get install curl jq

      # TODO: How can we avoid these duplicate lines from the enterprise system tests
    - name: Extract git branch
      id: git_info
      run: |
        echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
    - name: Extract git hashes from upstream and latest version
      id: git_upstream_info
      run: |
        echo "mlrun_hash=$( \
          cd /tmp && \
          git clone --single-branch --branch development https://github.com/mlrun/mlrun.git mlrun-upstream 2> /dev/null && \
          cd mlrun-upstream && \
          git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
          cd .. && \
          rm -rf mlrun-upstream)" >> $GITHUB_OUTPUT
        echo "ui_hash=$( \
          cd /tmp && \
          git clone --single-branch --branch development https://github.com/mlrun/ui.git mlrun-ui 2> /dev/null && \
          cd mlrun-ui && \
          git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
          cd .. && \
          rm -rf mlrun-ui)" >> $GITHUB_OUTPUT
        echo "unstable_version_prefix=$(cat automation/version/unstable_version_prefix)" >> $GITHUB_OUTPUT
    - name: Set computed versions params
      id: computed_params
      run: |
        action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \
        upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \
        export mlrun_hash=${action_mlrun_hash:-`echo $upstream_mlrun_hash`}
        echo "mlrun_hash=$(echo $mlrun_hash)" >> $GITHUB_OUTPUT
        action_mlrun_ui_hash=${{ steps.git_action_ui_info.outputs.ui_hash }} && \
        upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \
        export ui_hash=${action_mlrun_ui_hash:-`echo $upstream_mlrun_ui_hash`}
        echo "ui_hash=$(echo $ui_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_version=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}+$mlrun_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_docker_tag=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$mlrun_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_ui_version=${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$ui_hash" >> $GITHUB_OUTPUT
        echo "mlrun_docker_repo=$( \
          input_docker_repo=$INPUT_DOCKER_REPO && \
          echo ${input_docker_repo:-mlrun})" >> $GITHUB_OUTPUT
        echo "mlrun_docker_registry=$( \
          input_docker_registry=$INPUT_DOCKER_REGISTRY && \
          echo ${input_docker_registry:-ghcr.io/})" >> $GITHUB_OUTPUT
        echo "mlrun_system_tests_clean_resources=$( \
          input_system_tests_clean_resources=$INPUT_CLEAN_RESOURCES_IN_TEARDOWN && \
          echo ${input_system_tests_clean_resources:-true})" >> $GITHUB_OUTPUT
      env:
        INPUT_DOCKER_REPO: ${{ github.event.inputs.docker_repo }}
        INPUT_DOCKER_REGISTRY: ${{ github.event.inputs.docker_registry }}
        INPUT_CLEAN_RESOURCES_IN_TEARDOWN: ${{ github.event.inputs.clean_resources_in_teardown }}

    - uses: azure/setup-helm@v3
      with:
        version: "v3.9.1"

    - uses: manusa/actions-setup-minikube@v2.7.2
      with:
        minikube version: "v1.28.0"
        kubernetes version: "v1.23.9"
        driver: docker
        github token: ${{ github.token }}
        # I couldn't find a way to configure the IP (https://github.com/kubernetes/minikube/issues/951)
        # but this seems to work
        start args: '--addons=registry --insecure-registry="192.168.49.2:5000"'

    - name: Install MLRun CE helm chart
      run: |
        # TODO: There are a couple of modifications to the helm chart that we are doing right now:
        #       1. The grafana prometheus stack is disabled as there are currently no system tests checking its
        #          functionality. Once the model monitoring feature is complete and we have system tests for it, we
        #          can enable it.
        #       2. The mlrun DB is set as the old SQLite db. There is a bug in github workers when trying to run a mysql
        #          server pod in minikube installed on the worker, the mysql pod crashes. There isn't much information
        #          about this issue online as this isn't how github expect you to use mysql in workflows - the worker
        #          has a mysql server installed directly on it and should be enabled and used as the DB. So we might
        #          want in the future to use that instead, unless the mysql will be able to come up without crashing.
        # 
        # TODO: Align the mlrun config env vars with the ones in the prepare.py script to avoid further inconsistencies.
        python automation/deployment/ce.py deploy \
            --verbose \
            --minikube \
            --namespace=${NAMESPACE} \
            --registry-secret-name="" \
            --disable-prometheus-stack \
            --sqlite /mlrun/db/mlrun.db \
            --override-mlrun-api-image="${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_docker_tag }}" \
            --override-mlrun-ui-image="ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}" \
            --set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__BUILDER__MLRUN_VERSION_SPECIFIER="mlrun[complete] @ git+https://github.com/mlrun/mlrun@${{ steps.computed_params.outputs.mlrun_hash }}"' \
            --set mlrun.api.extraEnvKeyValue.MLRUN_IMAGES_REGISTRY="${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
            --set mlrun.api.extraEnvKeyValue.MLRUN_LOG_LEVEL="DEBUG" \
            --set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__SCHEDULING__MIN_ALLOWED_INTERVAL="0 seconds"' \
            --set mlrun.api.extraEnvKeyValue.MLRUN_MODEL_ENDPOINT_MONITORING__PARQUET_BATCHING_MAX_EVENTS="100"

    - name: Prepare system tests env
      run: |
        python automation/system_test/prepare.py env \
          --mlrun-dbpath "http://$(minikube ip):${MLRUN_API_NODE_PORT}" \
          --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}"

    # Enable tmate debugging of manually-triggered workflows if the input option was provided
    - name: Setup tmate session
      uses: mxschmitt/action-tmate@v3
      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
      with:

        # run in detach mode to allow the workflow to continue running while session is active
        # this will wait up to 10 minutes AFTER the entire job is done. Once user connects to the session,
        # it will wait until the user disconnects before finishing up the job.
        detached: true

    - name: Run system tests
      timeout-minutes: 180
      run: |
        MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ steps.computed_params.outputs.mlrun_system_tests_clean_resources }}" \
        MLRUN_VERSION="${{ steps.computed_params.outputs.mlrun_version }}" \
          make test-system-open-source

    - name: Output some logs in case of failure
      if: ${{ failure() }}
      # add set -x to print commands before executing to make logs reading easier
      run: |
        set -x
        minikube ip
        minikube logs
        minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1
        minikube kubectl -- --namespace ${NAMESPACE} get all
        minikube kubectl -- --namespace ${NAMESPACE} get all -o yaml
        minikube kubectl -- --namespace ${NAMESPACE} describe pods
        minikube kubectl -- --namespace ${NAMESPACE} get cm
        minikube kubectl -- --namespace ${NAMESPACE} get cm -o yaml
        minikube kubectl -- --namespace ${NAMESPACE} get secrets
        minikube kubectl -- --namespace ${NAMESPACE} get secrets -o yaml
        minikube kubectl -- --namespace ${NAMESPACE} get pvc
        minikube kubectl -- --namespace ${NAMESPACE} get pv
        set +x