embed controller into istio

Signed-off-by: spacewander <[email protected]>

.github/workflows/lint.yml

@@ -16,6 +16,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5

.github/workflows/test.yml

@@ -28,6 +28,8 @@ jobs:
         working-directory: ./api
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -70,6 +72,8 @@ jobs:
         working-directory: ./types
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -95,6 +99,8 @@ jobs:
         working-directory: ./plugins
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -118,6 +124,8 @@ jobs:
         working-directory: ./plugins
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -159,6 +167,8 @@ jobs:
         working-directory: ./controller
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -191,6 +201,8 @@ jobs:
     if: always()
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
     - name: Download artifact
       uses: actions/download-artifact@v4
     - name: Upload to codecov
@@ -213,6 +225,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -230,6 +244,8 @@ jobs:
         working-directory: ./e2e
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Go
       uses: actions/setup-go@v5
@@ -250,12 +266,26 @@ jobs:
 
     - name: Prepare cluster
       run: |
-        touch ../go.work.sum # work around the requirement of sum file in the commands below
-        make delete-cluster create-cluster deploy-cert-manager e2e-prepare-data-plane-image deploy-istio
+        make delete-cluster create-cluster
+
+    - name: Prepare code
+      run: |
+        pushd ..
+        touch go.work.sum # work around the requirement of sum file
+        make prebuild
+        popd
 
     - name: Prepare controller
       run: |
         make e2e-build-controller-image deploy-controller
 
+    - name: Prepare data plane
+      run: |
+        make e2e-prepare-data-plane-image
+
+    - name: Bring up istio
+      run: |
+        make deploy-istio
+
     - name: Test
       run: make run-e2e

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "istio"]
+	path = external/istio
+	url = [email protected]:istio/istio

.golangci.yml

@@ -46,12 +46,6 @@ issues:
         - errcheck
         - gosec
         - unparam
-    - path: e2e/ # e2e tests
-      linters:
-        - bodyclose
-        - errcheck
-        - gosec
-        - unparam
   # Show the complete output
   max-issues-per-linter: 0
   max-same-issues: 0

.licenserc.yaml

@@ -27,10 +27,12 @@ header:
     # ignore file controlled by tools
     - '**/go.sum'
     - 'buf.lock'
+    - '.gitmodules'
     # files can't add license
     - '**/*.pb.validate.go'
     - '**/*.json'
     - '**/*.txt'
+    - '**/*.patch'
     - 'plugins/VERSION'
     - '.ignore_words'
     # files from other projects

Makefile

@@ -12,10 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-include etc/common.mk
-
-GO_PROD_MODULES = api types controller plugins
-GO_MODULES = $(GO_PROD_MODULES) e2e
+include common.mk
 
 # For some tools, like golangci-lint, we prefer to use the latest version so that we can have the new feature.
 # For the other tools, like kind, we don't upgrade it until there is a strong reason.
@@ -44,7 +41,7 @@ gen-proto: dev-tools install-go-fmtter $(GO_TARGETS)
 .PHONY: gen-crd-code
 gen-crd-code: $(LOCALBIN) install-go-fmtter
 	test -s $(LOCALBIN)/client-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/[email protected]
-	LOCALBIN=$(LOCALBIN) etc/gen-crd-code.sh
+	LOCALBIN=$(LOCALBIN) tools/gen-crd-code.sh
 	$(LOCALBIN)/gosimports -w -local ${PROJECT_NAME} ./types/pkg/client
 
 .PHONY: dev-tools
@@ -79,17 +76,19 @@ lint-go:
 
 .PHONY: fmt-go
 fmt-go: install-go-fmtter
-	$(LOCALBIN)/gosimports -w -local ${PROJECT_NAME} .
 	$(foreach PKG, $(GO_MODULES), \
-		pushd ./${PKG} && go mod tidy || exit 1; popd; \
+		pushd ./${PKG} && \
+			go mod tidy || exit 1; \
+			$(LOCALBIN)/gosimports -w -local ${PROJECT_NAME} . || exit 1; \
+		popd; \
 	)
 
 # Don't use `buf format` to format the protobuf files! Buf's code style is different from Envoy.
 # That will break lots of things.
 .PHONY: lint-proto
 lint-proto: $(LOCALBIN)
 	test -x $(LOCALBIN)/buf || GOBIN=$(LOCALBIN) go install github.com/bufbuild/buf/cmd/[email protected]
-	$(LOCALBIN)/buf lint
+	$(LOCALBIN)/buf lint --exclude-path ./external
 
 .PHONY: fmt-proto
 fmt-proto: dev-tools
@@ -99,7 +98,7 @@ fmt-proto: dev-tools
 
 .PHONY: fmt-proto-local
 fmt-proto-local:
-	find . -name '*.proto' -exec clang-format -i {} \+
+	find . -name '*.proto' | grep -v './external' | xargs clang-format -i
 
 .PHONY: install-license-checker
 install-license-checker: $(LOCALBIN)
@@ -120,7 +119,7 @@ lint-spell: dev-tools
 		${DEV_TOOLS_IMAGE} \
 		make lint-spell-local
 
-CODESPELL = codespell --skip '.git,.idea,test-envoy,go.mod,go.sum,go.work.sum,*.svg,./site/public/**' --check-filenames --check-hidden --ignore-words ./.ignore_words
+CODESPELL = codespell --skip 'test-envoy,go.mod,go.sum,*.patch,*.svg,./site/public/**' --check-filenames --check-hidden --ignore-words ./.ignore_words $(shell ls -A | tr '\t' '\n' | grep -vE 'external|.git|.idea|go.work.sum')
 .PHONY: lint-spell-local
 lint-spell-local:
 	$(CODESPELL)
@@ -155,3 +154,11 @@ fmt: fmt-go fmt-proto
 .PHONY: verify-example
 verify-example:
 	cd ./examples/dev_your_plugin && ./verify.sh
+
+TARGET_ISTIO_DIR = $(shell pwd)/external/istio
+
+.PHONY: prebuild
+prebuild:
+	git submodule update --init --recursive
+	cd $(TARGET_ISTIO_DIR) && git status | grep -q "nothing to commit, working tree clean" || (echo "istio submodule is not clean, please commit your changes first"; exit 1)
+	cd ./patch && ./apply-patch.sh $(TARGET_ISTIO_DIR)

api/Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-include ../etc/common.mk
+include ../common.mk
 
 .PHONY: unit-test
 unit-test:

etc/common.mk → common.mk

@@ -34,11 +34,14 @@ GIT_VERSION     = $(shell git log -1 --pretty=format:%h)
 
 MIN_K8S_VERSION = 1.26.0
 
+GO_PROD_MODULES = api types controller plugins
+GO_MODULES = $(GO_PROD_MODULES) e2e
+
 # Define a recursive wildcard function
 rwildcard=$(foreach d,$(wildcard $(addsuffix *,$(1))),$(call rwildcard,$d/,$(2))$(filter $(subst *,%,$(2)),$d))
 
 PROTOC = protoc
-PROTO_FILES = $(call rwildcard,./,*.proto)
+PROTO_FILES = $(call rwildcard,$(GO_MODULES),*.proto)
 GO_TARGETS = $(patsubst %.proto,%.pb.go,$(PROTO_FILES))
 
 TEST_OPTION ?= -gcflags="all=-N -l" -race -covermode=atomic -coverprofile=cover.out -coverpkg=${PROJECT_NAME}/...

controller/Dockerfile

@@ -32,19 +32,10 @@ COPY api/ api/
 COPY types/ types/
 COPY plugins/ plugins/
 COPY controller/ controller/
-# we can't cache deps before building and copying source because each source directory has its "go.mod"
-RUN go mod download
-
-# Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN cd controller && CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /workspace/manager cmd/main.go
+# Remember to run `make prebuild` before building the image
+COPY external/istio/ external/istio/
+RUN cd ./external/istio && \
+    CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -C pilot/cmd/pilot-discovery -a -o /workspace/pilot-discovery
 
 FROM ${CONTROLLER_BASE_IMAGE}
-WORKDIR /
-COPY --from=builder /workspace/manager .
-USER 65532:65532
-
-ENTRYPOINT ["/manager"]
+COPY --from=builder /workspace/pilot-discovery /usr/local/bin/

controller/Makefile

@@ -15,9 +15,7 @@
 PROJECT_NAME    = mosn.io/htnn
 # Image URL to use all building/pushing image targets
 IMG ?= htnn-controller:latest
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-CONTROLLER_BASE_IMAGE ?= gcr.io/distroless/static:nonroot
+CONTROLLER_BASE_IMAGE ?= docker.io/istio/pilot:1.21.0
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.28.0
@@ -70,7 +68,7 @@ help: ## Display this help.
 
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd:ignoreUnexportedFields=true webhook paths="../types/..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd:ignoreUnexportedFields=true paths="../types/..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -148,7 +146,6 @@ deploy-image:
 
 .PHONY: deploy
 deploy: deploy-image manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
 .PHONY: undeploy

controller/cmd/main.go

@@ -29,14 +29,14 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
-	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	pkgLog "mosn.io/htnn/api/pkg/log"
 	"mosn.io/htnn/controller/internal/config"
 	"mosn.io/htnn/controller/internal/controller"
+	"mosn.io/htnn/controller/internal/controller/component"
+	"mosn.io/htnn/controller/internal/gatewayapi"
 	"mosn.io/htnn/controller/internal/log"
 	"mosn.io/htnn/controller/internal/registry"
-	"mosn.io/htnn/controller/internal/webhook"
 	v1 "mosn.io/htnn/types/apis/v1"
 )
 
@@ -78,8 +78,7 @@ func main() {
 	utilruntime.Must(istioscheme.AddToScheme(scheme))
 
 	if config.EnableGatewayAPI() {
-		// For HTTPRoute & Gateway, we only support v1 version
-		utilruntime.Must(gwapiv1.AddToScheme(scheme))
+		utilruntime.Must(gatewayapi.AddToScheme(scheme))
 	}
 
 	unsafeDisableDeepCopy := true
@@ -110,56 +109,33 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err = (&controller.HTTPFilterPolicyReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-	}).SetupWithManager(mgr); err != nil {
+	output := component.NewK8sOutput(mgr.GetClient())
+	rm := component.NewK8sResourceManager(mgr.GetClient())
+	if err = controller.NewHTTPFilterPolicyReconciler(
+		output,
+		rm,
+	).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "HTTPFilterPolicy")
 		os.Exit(1)
 	}
 	if err = (&controller.ConsumerReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		ResourceManager: rm,
+		Output:          output,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Consumer")
 		os.Exit(1)
 	}
 
 	registry.InitRegistryManager(&registry.RegistryManagerOption{
-		Client: mgr.GetClient(),
+		Output: output,
 	})
 	if err = (&controller.ServiceRegistryReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		ResourceManager: rm,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ServiceRegistry")
 		os.Exit(1)
 	}
 
-	if config.EnableWebhooks() {
-		if err = (&webhook.HTTPFilterPolicyWebhook{
-			HTTPFilterPolicy: v1.HTTPFilterPolicy{},
-		}).SetupWebhookWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create webhook", "webhook", "HTTPFilterPolicy")
-			os.Exit(1)
-		}
-
-		webhook.RegisterVirtualServiceWebhook(mgr)
-
-		if err = (&webhook.ConsumerWebhook{
-			Consumer: v1.Consumer{},
-		}).SetupWebhookWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create webhook", "webhook", "Consumer")
-			os.Exit(1)
-		}
-
-		if err = (&webhook.ServiceRegistryWebhook{
-			ServiceRegistry: v1.ServiceRegistry{},
-		}).SetupWebhookWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create webhook", "webhook", "ServiceRegistry")
-			os.Exit(1)
-		}
-	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {