build.yml

name: Build

on:
  push:
    branches-ignore:
      - 'site'
    tags:
      - '*'
  pull_request:
    branches:
      - master
      - 'releases/**'

env:
  GO_VERSION: 1.19
  # the newest supported K8s version and its respective kind image
  NEW_K8S_VERSION: v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1
  # the oldest supported K8s version and its respective kind image
  OLD_K8S_VERSION: v1.23.13@sha256:ef453bb7c79f0e3caba88d2067d4196f427794086a7d0df8df4f019d5e336b61
  LATEST_BACKWARD_SUPPORTED_VERSION: 0.7.0

jobs:
  verify:
    name: Verify
    runs-on: ubuntu-latest
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - name: Install Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.GO_VERSION }}
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Cache dependencies
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ~/go/pkg/mod
          ${{ github.workspace }}/hack/tools/bin
          ${{ github.workspace }}/hack/tools/lib
        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
    - name: Install tools
      run: make install-tools
    - name: Verify
      run: |
        make generate manifests generate-docs reconcile-requirements verify
        git diff --exit-code
  test:
    name: Unit tests
    runs-on: ubuntu-latest
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - name: Install Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.GO_VERSION }}
    - uses: actions/checkout@v3
    - name: Cache dependencies
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ~/go/pkg/mod
          ${{ github.workspace }}/hack/tools/bin
          ${{ github.workspace }}/hack/tools/lib
        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
    - name: Install tools
      run: make install-tools
    - name: Test
      run: make test

  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-latest
    env:
      DOCKER_HOSTNAME: "localhost:5000"
      DOCKER_NAMESPACE: fybrik-system
    strategy:
      matrix:
        k8s_version: [ 'new', 'old' ]
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - name: Install Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.GO_VERSION }}
    - uses: actions/checkout@v3
    - name: Cache dependencies
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ~/go/pkg/mod
          ${{ github.workspace }}/hack/tools/bin
          ${{ github.workspace }}/hack/tools/lib
        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
    - name: Install tools
      run: make install-tools
    - name: Namescope integration tests
      if: ${{ matrix.k8s_version == 'new' }}
      env:
        K8S_VERSION: ${{ env.NEW_K8S_VERSION }}
      run: make run-namescope-integration-tests
    - name: Integration tests old
      if: ${{ matrix.k8s_version == 'old' && ! github.event.pull_request.draft }}
      env:
        K8S_VERSION: ${{ env.OLD_K8S_VERSION }}
      run: make run-integration-tests
    - name: Integration tests new
      if: ${{ matrix.k8s_version == 'new' }}
      env:
        K8S_VERSION: ${{ env.NEW_K8S_VERSION }}
      run: make run-integration-tests
    - run: docker images
    - name: Save images
      if: ${{ github.event_name != 'pull_request' }}
      run: make save-images
    - name: Upload images
      if: ${{ github.event_name != 'pull_request' }}
      uses: actions/upload-artifact@v3
      with:
        name: images
        path: images.tar
        retention-days: 1

  notebook-sample-read-flow-old:
    name: Read flow notebook sample (old)
    if: ${{ ! github.event.pull_request.draft }}
    runs-on: ubuntu-latest
    env:
      DOCKER_HOSTNAME: "localhost:5000"
      DOCKER_NAMESPACE: fybrik-system
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - name: Install Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.GO_VERSION }}
    - uses: actions/checkout@v3
    - name: Cache dependencies
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ~/go/pkg/mod
          ${{ github.workspace }}/hack/tools/bin
          ${{ github.workspace }}/hack/tools/lib
        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
    - name: Install tools
      run: make install-tools
    - name: Read flow notebook tests old
      env:
        K8S_VERSION: ${{ env.OLD_K8S_VERSION }}
      run: make run-notebook-readflow-tests

#  notebook-backward-sample-read-flow:
#    name: Read flow notebook backward sample
#    runs-on: ubuntu-latest
#    env:
#      DOCKER_HOSTNAME: "localhost:5000"
#      DOCKER_NAMESPACE: fybrik-system
#    steps:
#    - name: Install Go
#      uses: actions/setup-go@v3
#      with:
#        go-version: ${{ env.GO_VERSION }}
#    - uses: actions/checkout@v3
#    - name: Cache dependencies
#      uses: actions/cache@v3
#      with:
#        path: |
#          ~/go/pkg/mod
#          ${{ github.workspace }}/hack/tools/bin
#          ${{ github.workspace }}/hack/tools/lib
#        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
#    - name: Install tools
#      run: make install-tools
#    - name: Read flow notebook tests bc version
#      env:
#        K8S_VERSION: ${{ env.NEW_K8S_VERSION }}
#        LATEST_BACKWARD_SUPPORTED_AFM_VERSION: v${{ env.LATEST_BACKWARD_SUPPORTED_VERSION }}
#        LATEST_BACKWARD_SUPPORTED_CRD_VERSION: ${{ env.LATEST_BACKWARD_SUPPORTED_VERSION }}
#      run: make run-notebook-readflow-bc-tests
   
  notebook-sample-flow:
    name: Notebook sample flows (new)
    runs-on: ubuntu-latest
    env:
      DOCKER_HOSTNAME: "localhost:5000"
      DOCKER_NAMESPACE: fybrik-system
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - run: echo "K8S_VERSION=${{ env.NEW_K8S_VERSION }}" >> $GITHUB_ENV
    - name: Install Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.GO_VERSION }}
    - uses: actions/checkout@v3
    - name: Cache dependencies
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ~/go/pkg/mod
          ${{ github.workspace }}/hack/tools/bin
          ${{ github.workspace }}/hack/tools/lib
        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
    - name: Install tools
      run: make install-tools
    - name: Write flow notebook tests
      run: make run-notebook-writeflow-tests
    - name: Update env vars to use existsing cluster
      run: |
        # The step that creates or updates the environment variable does
        # not have access to the new value, but all subsequent steps 
        # in a job will have access.
        echo "DEPLOY_OPENMETADATA_SERVER=0" >> $GITHUB_ENV
        echo "USE_EXISTING_CLUSTER=1" >> $GITHUB_ENV
    - name: Clean cluster
      run: |
        make undeploy-fybrik
        make clean-cluster-prepare
        killall kubectl || true
    - name: Read flow notebook tests
      run: make run-notebook-readflow-tests
    - name: Clean cluster
      run: |
        make undeploy-fybrik
        make clean-cluster-prepare
        killall kubectl || true
    - name: Read flow notebook tests with tls
      run: make run-notebook-readflow-tls-tests
    - name: Clean cluster
      run: |
        make undeploy-fybrik
        make clean-cluster-prepare
        killall kubectl || true
    - name: Read flow notebook tests with tls using system CA certs
      run: make run-notebook-readflow-tls-system-cacerts-tests
    - name: Clean cluster
      if: ${{ ! github.event.pull_request.draft }}
      run: |
        make undeploy-fybrik
        make clean-cluster-prepare
        killall kubectl || true
    - name: Read flow notebook tests with katalog
      if: ${{ ! github.event.pull_request.draft }}
      run: make run-notebook-readflow-tests-katalog


  tekton-pipelines:
    if: ${{ ! github.event.pull_request.draft }}
    name: Tekton Tests
    runs-on: ubuntu-latest
    env:
      DOCKER_HOSTNAME: "localhost:5000"
      DOCKER_NAMESPACE: fybrik-system
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - name: Install Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.GO_VERSION }}
    - uses: actions/checkout@v3
    - name: install nfs rpms
      run: sudo apt-get update && sudo apt-get install -y nfs-kernel-server nfs-common && sudo modprobe nfsd && sudo modprobe nfs
    - name: Cache dependencies
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ~/go/pkg/mod
          ${{ github.workspace }}/hack/tools/bin
          ${{ github.workspace }}/hack/tools/lib
        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.env') }}-go-${{ hashFiles('**/go.sum') }}
    - name: Install tools
      run: make install-tools
    - name: Integration tests
      env:
        image_source_repo_password: "fake"
        github: github.com
        run_tkn: 1
        dockerhub_hostname: docker.io
        git_user: fake@fake.com
        image_repo: kind-registry:5000
        image_source_repo_username: fake@fake.com
        GH_TOKEN: fake
      run: pushd hack/tools/ && ./create_kind.sh && docker info && popd && . pipeline/source-external.sh && skip_tests=false kind=true github_workspace=${{ github.workspace }} pipeline/bootstrap-pipeline.sh fybrik-system

  push_images:
    name: Push images
    runs-on: ubuntu-latest
    env:
      DOCKER_HOSTNAME: "localhost:5000"
      DOCKER_NAMESPACE: fybrik-system
      DOCKER_USERNAME: ${{ github.actor }}
      DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
    needs: [verify, test, integration-tests]
    if: ${{ github.event_name != 'pull_request' && github.repository == 'fybrik/fybrik' }}
    steps:
    - name: Set Swap Space
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10
    - uses: actions/checkout@v3
    - name: Download artifact
      uses: actions/download-artifact@v3
      with:
        name: images
    - name: Load images
      run: docker load -i images.tar
    - run: docker images
    # Versions are created as follows (In helm 3.7 and above OCI reference tags
    # must be valid SemVer):
    # - Tags starting with v will be released as what comes after `v`. (e.g. refs/tags/v1.0 -> 1.0).
    # - Master branch will be released with `master` as version tag for the docker images and
    #   `0.0.0-master` tag for helm chart.
    - id: version
      name: Infer version
      run: |
        publish_images='false'
        if [[ ${GITHUB_REF} == refs/tags/* ]] ;
        then
            publish_images='true'
            version="${GITHUB_REF#refs/tags/v}"
            echo "HELM_TAG=${version}" >> $GITHUB_ENV
        elif  [[ ${GITHUB_REF} == refs/heads/master ]] ;
        then
            publish_images='true'
            version=master
            echo "HELM_TAG=0.0.0-master" >> $GITHUB_ENV
        fi
        echo ::set-output name=version::$version
        echo ::set-output name=publish_images::$publish_images
    - name: Publish images
      if: ${{ steps.version.outputs.publish_images == 'true' }}
      env:
        DOCKER_PUBLIC_TAGNAME:  ${{ steps.version.outputs.version }}
      run: make docker-retag-and-push-public && make helm-push-public