pipeline.yml

# MIT License
#
# Copyright (c) 2021 Theodor Amariucai and EASE Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

name: STeLLAR Pipeline

concurrency: staging_environment

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  build_client:
    name: Build framework
    runs-on: ubuntu-22.04
    env:
      working-directory: ./src
    steps:

      - name: Check out code into the Go module directory
        uses: actions/checkout@v2

      - name: Check syntax
        uses: golangci/golangci-lint-action@v2.5.2
        with:
          working-directory: ${{env.working-directory}}
          args: --timeout 5m --skip-dirs raw-code
          skip-pkg-cache: true
          skip-build-cache: true

      - name: Build client binary
        working-directory: ${{env.working-directory}}
        run: |
          docker build -t stellar .
          docker create --name temp stellar /bin/bash

      - name: Package client artifact
        run: |
          mkdir -p "setup/deployment/raw-code"
          docker cp temp:/app/stellar ./main
          docker cp temp:/app/setup/deployment/raw-code/functions setup/deployment/raw-code/functions
          tar -czvf build.tar ./main ./setup/deployment/raw-code/functions

      - name: Upload client artifact
        uses: actions/upload-artifact@v2
        with:
          name: STeLLAR-build
          path: build.tar
          retention-days: 1

  test_vhive:
    name: vHive integration tests
    needs: [ build_client ]
    runs-on: ubuntu-22.04
    env:
      working-directory: ./src
    steps:

      - name: Check out code into the Go module directory
        uses: actions/checkout@v2

      - name: Download client artifact
        uses: actions/download-artifact@v2
        with:
          name: STeLLAR-build

      - name: Untar client build
        run: tar --strip-components=1 -xvf build.tar

      - name: Setup vHive Data Transfer (producer-consumer) Containers
        run: |
          cat > vHive.json <<- EOM
          [
          {
              "GatewayID": "10.96.0.2:50051",
              "FunctionMemoryMB": 128,
              "ImageSizeMB": 60,
              "PackageType": "Image"
          },
          {
              "GatewayID": "10.96.0.3:50051",
              "FunctionMemoryMB": 128,
              "ImageSizeMB": 60,
              "PackageType": "Image"
          }
          ]
          EOM
          docker network create --subnet=10.96.0.0/16 benchnet
          docker run -d --name=producer --net=benchnet --ip 10.96.0.2 vhiveease/producer-consumer:latest
          docker run -d --name=consumer --net=benchnet --ip 10.96.0.3 vhiveease/producer-consumer:latest
          docker run -d -p 50052:9000 --net=benchnet --ip 10.96.0.46 --name minio -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" minio/minio server /data
          wget https://dl.min.io/client/mc/release/linux-amd64/mc
          chmod +x mc
          sudo mv mc /usr/local/bin
          mc alias set myminio http://10.96.0.46:9000 minio minio123
          mc mb myminio/mybucket

      - name: vHive Producer-consumer (Inline Data Transfer) Integration Tests
        run: ./main --g . --o latency-samples --c experiments/tests/vhive/data-transfer.json

      - name: vHive Producer-consumer (Storage Data Transfer) Integration Tests
        run: ./main --g . --o latency-samples --c experiments/tests/vhive/data-transfer-minio.json

      - name: vHive Chameleon Integration Tests
        run: |
          cat > vHive.json <<- EOM
          [
          {
              "GatewayID": "10.96.0.4:50051",
              "FunctionMemoryMB": 128,
              "ImageSizeMB": 60,
              "PackageType": "Image"
          }
          ]
          EOM
          docker run -d --name=chameleon-func --net=benchnet --ip 10.96.0.4 vhiveease/chameleon:latest
          ./main --g . --o latency-samples --c experiments/tests/vhive/burstiness-image-package.json

      - name: vHive Hello Python Integration Tests
        run: |
          cat > vHive.json <<- EOM
          [
          {
              "GatewayID": "10.96.0.5:50051",
              "FunctionMemoryMB": 128,
              "ImageSizeMB": 60,
              "PackageType": "Image"
          }
          ]
          EOM
          docker run -d --name=hellopy-func --net=benchnet --ip 10.96.0.5 vhiveease/hellopy:latest
          ./main --g . --o latency-samples --c experiments/tests/vhive/burstiness-image-package.json

      - name: vHive RNN Serving Integration Tests
        run: |
          cat > vHive.json <<- EOM
          [
          {
              "GatewayID": "10.96.0.6:50051",
              "FunctionMemoryMB": 128,
              "ImageSizeMB": 60,
              "PackageType": "Image"
          }
          ]
          EOM
          docker run -d --name=rnnserving-func --net=benchnet --ip 10.96.0.6 vhiveease/rnnserving:latest
          ./main --g . --o latency-samples --c experiments/tests/vhive/burstiness-image-package.json

  deploy:
    name: Deploy framework
    needs: [ test_aws, test_vhive ]
    runs-on: ubuntu-22.04
    steps:

      - name: Check out code into the Go module directory
        uses: actions/checkout@v2

      - name: Download client build
        uses: actions/download-artifact@v2
        with:
          name: STeLLAR-build

      - name: Untar client build
        run: tar --strip-components=1 -xvf build.tar

      - name: Prepare benchmarking functions
        run: |
          mkdir -p setup/deployment/raw-code/functions
          cp -R ./src/setup/deployment/raw-code/functions setup/deployment/raw-code/functions

      - name: Push to deployment branch
        run: |
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config user.name "vHive Bench Bot"
          git remote set-url origin https://ease-lab:${{secrets.ACCESS_TOKEN}}@github.com/ease-lab/STeLLAR.git
          git checkout --orphan deployment
          git rm -rf --ignore-unmatch --cached .

          git add --ignore-errors --force scripts endpoints experiments main setup/deployment/raw-code/functions

          git commit -m "Automated push from successful STeLLAR pipeline"
          git push origin HEAD:deployment --force