[Monorepo] - Run components unit tests

.github/workflows/checks.yml

@@ -0,0 +1,73 @@
+name: Run unit tests
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+env:
+  GO_VERSION: "1.19"
+  PRIORITIES": "P0"
+jobs:
+  lint:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          # TODO(monorepo): Enable lint flytecopilot
+          # flytecopilot
+          - flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ matrix.component }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Lint
+        run: make install && make lint
+  unit-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          # TODO(monorepo): Enable flyteidl unit tests
+          # flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ matrix.component }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Unit Tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make install && make test_unit_codecov
+      # TODO(monorepo): investigate how codecoverage is aggregated
+      - name: Push CodeCov
+        uses: codecov/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          file: coverage.txt
+          flags: unittests
+          fail_ci_if_error: false

.github/workflows/flyteidl-verification.yml

@@ -0,0 +1,279 @@
+name: Verification Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  # These two jobs, lint and tests, run in checks.yml, alongside the other
+  # components.
+  #
+  # lint:
+  #   name: Lint
+  #   uses: flyteorg/flytetools/.github/workflows/lint.yml@master
+  #   with:
+  #     go-version: 1.19
+  # tests:
+  #   name: Unit Tests
+  #   uses: flyteorg/flytetools/.github/workflows/tests.yml@master
+  #   secrets:
+  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  #   with:
+  #     go-version: 1.19
+  generate-protos:
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    name: Generate Protos
+    container:
+      image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2
+      options: --cpus 1
+      volumes:
+        - /github/workspace:/defs
+      env:
+        REPO_BLOB_SHA: master
+        PROJECT_ANNOTATION_PREFIX: flyte.interface
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      # GO Protos
+      - name: Proto-Service-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service --with_gateway -l go --go_source_relative
+      - name: Proto-Admin-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin --with_gateway -l go --go_source_relative --validate_out
+      - name: Proto-Core-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core --with_gateway -l go --go_source_relative --validate_out
+      - name: Proto-Event-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event --with_gateway -l go --go_source_relative --validate_out
+      - name: Proto-Plugins-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l go --go_source_relative --validate_out
+      - name: Proto-Datacatalog-Go
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l go --go_source_relative --validate_out
+
+      # Python is going to be generated using buf
+      # Python
+      # - name: Proto-Service-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python
+      # - name: Proto-Admin-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python
+      # - name: Proto-Core-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python
+      # - name: Proto-Event-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python
+      # - name: Proto-Plugins-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python
+      # - name: Proto-Datacatalog-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python
+
+      # Cpp
+      - name: Proto-Service-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l cpp
+      - name: Proto-Admin-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l cpp
+      - name: Proto-Core-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l cpp
+      - name: Proto-Event-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l cpp
+      - name: Proto-Plugins-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l cpp
+      - name: Proto-Datacatalog-Cpp
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l cpp
+
+      # Java
+      - name: Proto-Service-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l java
+      - name: Proto-Admin-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l java
+      - name: Proto-Core-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l java
+      - name: Proto-Event-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l java
+      - name: Proto-Plugins-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l java
+      - name: Proto-Datacatalog-Java
+        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l java
+
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: ./gen
+
+  generate-protos-buf:
+    name: Generate python protos using buf
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    needs: [ generate-protos ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - uses: bufbuild/buf-setup-action@v1
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - name: "Run buf generate"
+        run: buf generate
+      - name: "Include __init__.py files"
+        run: find gen/pb_python -type d -exec touch {}/__init__.py \;
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated-buf
+          path: ./gen
+
+  generate-admin-swagger-code:
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    name: Generate Admin Swagger Code
+    needs:
+      - generate-protos
+      - generate-protos-buf
+    container:
+      image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2
+      options: --cpus 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated-buf
+          path: ./gen
+      - run: ls gen/pb-go/flyteidl/service/
+      - run: cp -R gen/pb-go/flyteidl/service/ ../
+      - run: ls ../service/
+      # Open API 2
+      - name: OpenAPI-Binary
+        run: go-bindata -pkg service -o ../service/openapi.go -prefix ../service/ -modtime 1562572800 ../service/admin.swagger.json
+      - run: rm -rf gen/pb-go/flyteidl/service
+      - run: mkdir -p gen/pb-go/flyteidl/service
+      - run: cp -R ../service gen/pb-go/flyteidl/
+      - run: ls gen/pb-go/flyteidl/
+      - run: ls gen/pb-go/flyteidl/service/
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: ./gen
+
+  generate-js-code:
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    name: Generate JS Code
+    needs: [ generate-admin-swagger-code ]
+    container:
+      image: schottra/docker-protobufjs:v0.0.2
+      options: --cpus 1
+      volumes:
+        - /github/workspace:/defs
+      env:
+        REPO_BLOB_SHA: master
+        PROJECT_ANNOTATION_PREFIX: flyte.interface
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - run: mkdir -p /defs
+      - run: mkdir -p /gen
+      - run: cp -R . /defs
+      - run: node /code/generate-protobuf.js --module-name flyteidl -d protos/flyteidl/core  -d protos/flyteidl/event -d protos/flyteidl/admin -d protos/flyteidl/service  -- --root flyteidl -t static-module -w default --no-delimited --force-long --no-convert -p /defs/protos
+      - run: ls /defs/gen/
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: /defs/gen
+
+  generate-swagger-code:
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    name: Generate Swagger Code
+    needs: [ generate-js-code ]
+    container:
+      image: docker.io/lyft/swagger-codegen-cli:dc5ce6ec6d7d4d980fa882d6bd13a83cba3be3c3
+      options: --cpus 1
+      volumes:
+        - /github/workspace:/defs
+      env:
+        REPO_BLOB_SHA: master
+        PROJECT_ANNOTATION_PREFIX: flyte.interface
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l go -o ./gen/pb-go/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin
+      - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l python -o ./gen/pb_python/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated
+          path: ./gen
+
+  test-proto-changes:
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    name: Test Proto Changes
+    needs: [ generate-swagger-code ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated
+          path: ./gen
+      - name: Test
+        run: ./scripts/test_diff.sh
+
+  generate:
+    defaults:
+      run:
+        working-directory: flyteidl
+    runs-on: ubuntu-latest
+    name: Go Generate
+    steps:
+      - uses: actions/checkout@v3
+      - uses: arduino/setup-protoc@v1
+      - uses: bufbuild/buf-setup-action@v1
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - name: Go generate and diff
+        run: DELTA_CHECK=true make generate

Dockerfile

@@ -9,6 +9,15 @@ ENV GOARCH "${TARGETARCH}"
 ENV GOOS linux
 
 WORKDIR /flyteorg/build
+
+COPY datacatalog datacatalog
+COPY flyteadmin flyteadmin
+COPY flytecopilot flytecopilot
+COPY flyteidl flyteidl
+COPY flyteplugins flyteplugins
+COPY flytepropeller flytepropeller
+COPY flytestdlib flytestdlib
+
 COPY go.mod go.sum ./
 RUN go mod download
 COPY cmd cmd

cmd/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/flyteorg/flyte/cmd/single"
-	_ "github.com/flyteorg/flytepropeller/plugins"
+	_ "github.com/flyteorg/flyte/flytepropeller/plugins"
 	"github.com/golang/glog"
 )
 

cmd/single/config.go

@@ -1,6 +1,6 @@
 package single
 
-import "github.com/flyteorg/flytestdlib/config"
+import "github.com/flyteorg/flyte/flytestdlib/config"
 
 //go:generate pflags Config --default-var=DefaultConfig
 
@@ -21,9 +21,9 @@ type Propeller struct {
 }
 
 type Admin struct {
-	Disabled                      bool `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
-	DisableScheduler              bool `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
-	DisableClusterResourceManager bool `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
+	Disabled                      bool     `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
+	DisableScheduler              bool     `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
+	DisableClusterResourceManager bool     `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
 	SeedProjects                  []string `json:"seedProjects" pflag:",flyte projects to create by default."`
 }