Monorepo build images

.github/workflows/checks.yml

@@ -0,0 +1,162 @@
+name: Components Checks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+env:
+  GO_VERSION: "1.19"
+  PRIORITIES: "P0"
+jobs:
+  unpack-envvars:
+    runs-on: ubuntu-latest
+    outputs:
+      go-version: ${{ steps.step.outputs.go-version }}
+    steps:
+      - id: step
+        run: |
+          echo "go-version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
+
+  lint:
+    name: Lint
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          # TODO(monorepo): Enable lint flytecopilot
+          # - flytecopilot
+          - flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    uses: ./.github/workflows/lint.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+  unit-tests:
+    name: Unit Tests
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          # TODO(monorepo): Enable flyteidl unit tests
+          # - flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  docker-build:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+    name: Docker Build Images
+    uses: ./.github/workflows/component_docker_build.yml
+    with:
+       component: ${{ matrix.component }}
+
+  # TODO(monorepo): these tests are broken. They never test an actual change.
+  # endtoend:
+  #   name: End2End Test
+  #   needs: [ docker-build ]
+  #   uses: ./.github/workflows/end2end.yml
+  #   with:
+  #     # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id)
+  #     cache_key: ${{ needs.docker-build.outputs.cache_key }}
+  #     priorities: "P0"
+
+  integration:
+    name: Integration Test
+    needs:
+      - docker-build
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - flyteadmin
+          - flytepropeller
+    uses: ./.github/workflows/integration.yml
+    with:
+      component: ${{ matrix.component }}
+      cache_key: ${{ needs.docker-build.outputs.cache_key }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+
+  generate:
+    name: Check Go Generate
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+    uses: ./.github/workflows/go_generate.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+
+  # TODO(monorepo): enable bump_version
+  # bump_version:
+  #   name: Bump Version
+  #   if: ${{ github.event_name != 'pull_request' }}
+  #   needs: [ endtoend,  integration, lint, tests, generate ] # Only to ensure it can successfully build
+  #   uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master
+  #   secrets:
+  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  # goreleaser:
+  #   name: Goreleaser
+  #   needs: [ bump_version ] # Only to ensure it can successfully build
+  #   uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
+  #   with:
+  #     go-version: "1.19"
+  #   secrets:
+  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  push_docker_image:
+    name: Build & Push Image
+    # TODO(monorepo): depend on bump_version
+    # needs: [ bump_version ]
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+          - flytescheduler
+    uses: ./.github/workflows/publish.yml
+    with:
+      version: "test-version-monorepo"
+      component: ${{ matrix.component }}
+      dockerfile: Dockerfile.${{ matrix.component }}
+      # TODO(monorepo): set push to true once we depend on bump_version
+      # push: true
+      push: false
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/component_docker_build.yml

@@ -0,0 +1,49 @@
+name: Build Docker Image
+
+on:
+  workflow_call:
+    inputs:
+      component:
+        required: true
+        type: string
+    outputs:
+      cache_key:
+        description: "Docker Cache key"
+        value: ${{ jobs.build_docker.outputs.cache_key }}
+jobs:
+  build_docker:
+    runs-on: ubuntu-latest
+    outputs:
+      cache_key: ${{ steps.cache_key.outputs.cache_key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-${{ inputs.component }}
+          key: /tmp/docker-images-${{ github.run_id }}-${{ inputs.component }}
+          restore-keys: |
+            /tmp/docker-images-
+      - name: Set cache key output
+        id: cache_key
+        run: |
+          echo ::set-output name=cache_key::/tmp/docker-images-${{ github.run_id }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        env:
+          # We are unable to leverage docker buildx here without changing the
+          # caching mechanism significantly. See here for the caching options
+          # available for docker buildx: https://docs.docker.com/engine/reference/commandline/buildx_build/#cache-from
+          # For now at least enable DOCKER_BUILDKIT for faster builds. Eventually we
+          # should rewrite this pipeline to use docker buildx with cache type=gha.
+          DOCKER_BUILDKIT: "1"
+        run: |
+          docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:builder --target builder --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} .
+          docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:latest --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} .
+
+      - name: Tag and cache docker image
+        run: |
+          mkdir -p /tmp/tmp/docker-images-${{ inputs.component }} && docker save ${{ github.repository_owner }}/${{ inputs.component }}:builder -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-builder-${{ inputs.component }}.tar && docker save ${{ github.repository_owner }}/${{ inputs.component }}:latest -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-${{ inputs.component }}.tar

.github/workflows/end2end.yml

@@ -0,0 +1,134 @@
+name: End to End tests
+
+on:
+  workflow_call:
+    inputs:
+      priorities:
+        description: "Priorities of tests to register (comma-separated)"
+        required: true
+        type: string
+      cache_key:
+        description: "Cache key for docker image"
+        required: true
+        type: string
+jobs:
+  endtoend:
+    name: End to End tests
+    runs-on: ubuntu-latest
+    env:
+      FLYTESNACKS_VERSION: ""
+    steps:
+      - name: Set latest Flytesnacks release
+        if: ${{ env.FLYTESNACKS_VERSION == '' }}
+        run: |
+          FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
+          echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: unionai/[email protected]
+        name: Setup flytectl
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.11
+      - id: load-docker-cache-datacatalog
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-datacatalog
+          key: ${{ inputs.cache_key }}-datacatalog
+      - id: load-docker-cache-flyteadmin
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-flyteadmin
+          key: ${{ inputs.cache_key }}-flyteadmin
+      - id: load-docker-cache-flytecopilot
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-flytecopilot
+          key: ${{ inputs.cache_key }}-flytecopilot
+      - id: load-docker-cache-flytepropeller
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-flytepropeller
+          key: ${{ inputs.cache_key }}-flytepropeller
+      - name: Create Sandbox Cluster
+        run: |
+          cp /tmp/tmp/docker-images-datacatalog/snapshot-datacatalog.tar snapshot-datacatalog.tar
+          cp /tmp/tmp/docker-images-flyteadmin/snapshot-flyteadmin.tar snapshot-flyteadmin.tar
+          cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar
+          cp /tmp/tmp/docker-images-flytepropeller/snapshot-flytepropeller.tar snapshot-flytepropeller.tar
+          flytectl config init
+          flytectl sandbox start --source=$(pwd)
+      - name: Prime docker cache
+        run: |
+          flytectl sandbox exec -- docker load -i /root/snapshot-datacatalog.tar
+          flytectl sandbox exec -- docker load -i /root/snapshot-flyteadmin.tar
+          flytectl sandbox exec -- docker load -i /root/snapshot-flytecopilot.tar
+          flytectl sandbox exec -- docker load -i /root/snapshot-flytepropeller.tar
+      - name: Setup Flytekit
+        run: |
+          python -m pip install --upgrade pip
+          pip install flytekit flytekitplugins-deck-standard
+          pip freeze
+      - name: Checkout flytesnacks
+        if: ${{ inputs.priorities == 'P0' }}
+        uses: actions/checkout@v3
+        with:
+          repository: flyteorg/flytesnacks
+          path: flytesnacks
+          ref: ${{ env.FLYTESNACKS_VERSION }}
+      - name: Register P0 tests
+        if: ${{ inputs.priorities == 'P0' }}
+        run: |
+          for f in \
+                  basics/basics/hello_world.py \
+                  basics/basics/workflow.py \
+                  basics/basics/named_outputs.py \
+                  advanced_composition/advanced_composition/chain_entities.py \
+                  advanced_composition/advanced_composition/dynamics.py \
+                  advanced_composition/advanced_composition/map_task.py \
+                  advanced_composition/advanced_composition/subworkflows.py \
+                  data_types_and_io/data_types_and_io/custom_objects.py \
+                  data_types_and_io/data_types_and_io/schema.py \
+                  data_types_and_io/data_types_and_io/typed_schema.py ;
+          do
+              pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
+                  register \
+                  --project flytesnacks \
+                  --domain development \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
+                  --version ${{ env.FLYTESNACKS_VERSION }} \
+                  flytesnacks/examples/$f;
+          done
+      - name: Register all flytesnacks examples
+        if: ${{ inputs.priorities != 'P0' }}
+        uses: unionai/[email protected]
+        with:
+          flytesnacks: true
+          project: flytesnacks
+          version: ${{ env.FLYTESNACKS_VERSION }}
+          domain: development
+      # - name: Pre Upgrade Tests
+      #   if: ${{ github.event.repository.name == 'flyteadmin' }}
+      #   env:
+      #     PRIORITIES: "${{ inputs.priorities }}"
+      #   run: |
+      #     make end2end_execute
+      - name: Upgrade Helm charts
+        run: |
+          flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte
+          flytectl sandbox exec -- helm repo update
+          flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest
+          # TODO(monorepo): the following commands are not correct.
+          # we have to separate the calls to replace the images into different commands.
+          # flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \
+          #   --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest
+          #   --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest
+          #   --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest
+          #   --set flytepropeller.image.repository=${{ github.repository_owner }}/datacalog,flytepropeller.image.tag=latest
+
+          flytectl sandbox exec -- k3s kubectl get pods -n flyte -oyaml
+      - name: Post Upgrade Tests
+        env:
+          PRIORITIES: "${{ inputs.priorities }}"
+        run: |
+          make end2end_execute

.github/workflows/go_generate.yml

@@ -0,0 +1,27 @@
+name: Go Generate
+
+on:
+  workflow_call:
+    inputs:
+      component:
+        required: true
+        type: string
+      go-version:
+        required: true
+        type: string
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    name: Go Generate
+    defaults:
+      run:
+        working-directory: ${{ inputs.component }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+      - uses: bufbuild/buf-setup-action@v1
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ inputs.go-version }}
+      - name: Go generate and diff
+        run: DELTA_CHECK=true make generate