Kubernetes Operator for Nessie

bom/build.gradle.kts

@@ -41,6 +41,7 @@ dependencies {
     api(project(":nessie-jaxrs-tests"))
     api(project(":nessie-keycloak-testcontainer"))
     api(project(":nessie-nessie-testcontainer"))
+    api(project(":nessie-operator"))
     api(project(":nessie-quarkus-auth"))
     api(project(":nessie-quarkus-common"))
     api(project(":nessie-quarkus-cli"))

gradle/libs.versions.toml

@@ -146,6 +146,7 @@ quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quark
 quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkus" }
 quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkus" }
 quarkus-logging-sentry = { module = "io.quarkiverse.loggingsentry:quarkus-logging-sentry", version = "2.0.5" }
+quarkus-operator-sdk-bom = { module = "io.quarkus.platform:quarkus-operator-sdk-bom", version.ref = "quarkus" }
 rest-assured = { module = "io.rest-assured:rest-assured", version = "5.4.0" }
 rocksdb-jni = { module = "org.rocksdb:rocksdbjni", version = "8.10.0" }
 scala-library-v212 = { module = "org.scala-lang:scala-library", version = { strictly = "[2.12, 2.13[", prefer = "2.12.18" }}

gradle/projects.main.properties

@@ -21,6 +21,7 @@ nessie-jaxrs-testextension=servers/jax-rs-testextension
 nessie-jaxrs-tests=servers/jax-rs-tests
 nessie-keycloak-testcontainer=testing/keycloak-container
 nessie-nessie-testcontainer=testing/nessie-container
+nessie-operator=operator
 nessie-quarkus-auth=servers/quarkus-auth
 nessie-quarkus-cli=servers/quarkus-cli
 nessie-quarkus-common=servers/quarkus-common

operator/Makefile

@@ -0,0 +1,110 @@
+
+VERSION ?= $(shell cat ../version.txt | sed -e 's/.*-SNAPSHOT/latest/g')
+RELEASE_VERSION ?= $(shell cat ../version.txt | sed -e 's/-SNAPSHOT//g')
+
+# CHANNELS define the bundle channels used in the bundle.
+# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
+# To re-generate a bundle for other specific channels without changing the standard setup, you can:
+# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
+# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
+ifneq ($(origin CHANNELS), undefined)
+BUNDLE_CHANNELS := --channels=$(CHANNELS)
+endif
+
+# DEFAULT_CHANNEL defines the default channel used in the bundle.
+# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
+# To re-generate a bundle for any other default channel without changing the default setup, you can:
+# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
+# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
+ifneq ($(origin DEFAULT_CHANNEL), undefined)
+BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
+endif
+BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
+
+# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
+# This variable is used to construct full image tags for bundle and catalog images.
+IMAGE_TAG_BASE ?= ghcr.io/projectnessie/nessie-operator
+
+# BUNDLE_IMG defines the image:tag used for the bundle.
+# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
+BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION)
+
+# Image URL to use all building/pushing image targets
+IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
+
+PULL_POLICY ?= $(shell [ "$(VERSION)" = "latest" ] && echo "Always" || echo "IfNotPresent")
+PLATFORM ?= linux/$(shell arch)
+
+all: docker-build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Build
+
+docker-build: ## Build docker image with the manager.
+	../gradlew --no-build-cache :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
+		-Dquarkus.container-image.image=${IMG} \
+		-Dquarkus.jib.platforms=${PLATFORM} \
+		-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}
+
+docker-push: ## Build and push docker image with the manager.
+	../gradlew --no-build-cache :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
+		-Dquarkus.container-image.push=true \
+		-Dquarkus.container-image.image=${IMG} \
+		-Dquarkus.jib.platforms=${PLATFORM} \
+		-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}
+
+##@ Deployment
+
+install: ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl apply -f $(file);)
+
+uninstall: ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl delete -f $(file);)
+
+deploy: ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	kubectl apply -f build/kubernetes/kubernetes.yml
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	kubectl delete -f build/kubernetes/kubernetes.yml
+
+##@ Helm
+
+helm-install: ## Install CRDs and the operator using Helm.
+	helm install nessie-operator build/helm -n nessie-operator
+
+helm-upgrade: ## Upgrade CRDs and the operator using Helm.
+	helm upgrade nessie-operator build/helm -n nessie-operator
+
+helm-uninstall: ## Uninstall CRDs and the operator using Helm.
+	helm uninstall nessie-operator -n nessie-operator
+
+##@ Bundle
+
+.PHONY: bundle
+bundle:  ## Generate bundle manifests and metadata, then validate generated files.
+	cat build/kubernetes/* | operator-sdk generate bundle -q --overwrite --version $(RELEASE_VERSION) $(BUNDLE_METADATA_OPTS)
+	operator-sdk bundle validate ./bundle
+	# TODO use quarkus
+
+.PHONY: bundle-build
+bundle-build: ## Build the bundle image.
+	docker build -f build/bundle/nessie-operator/bundle.Dockerfile -t $(BUNDLE_IMG) build/bundle/nessie-operator
+
+.PHONY: bundle-push
+bundle-push: ## Push the bundle image.
+	docker push $(BUNDLE_IMG)

operator/PROJECT

@@ -0,0 +1,17 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
+domain: projectnessie.org
+layout:
+- quarkus.javaoperatorsdk.io/v1-alpha
+projectName: nessie-operator
+resources:
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: projectnessie.org
+  group: nessie
+  kind: Nessie
+  version: v1alpha1
+version: "3"

operator/README.md

@@ -0,0 +1,142 @@
+# Kubernetes Operator for Nessie
+
+## Overview
+
+This module is a Kubernetes Operator for Nessie.
+
+**WARNING: This is a work in progress and is not ready for production use.**
+
+This project was created using [Operator SDK]:
+
+```bash
+operator-sdk init --plugins=quarkus --domain=projectnessie.org --project-name=nessie-operator
+operator-sdk create api --plugins=quarkus --version=v1alpha1 --kind=Nessie
+```
+
+[Operator SDK]:https://sdk.operatorframework.io/docs/cli/operator-sdk/
+
+## Usage
+
+TODO describe typical deployment scenarios.
+
+## Development
+
+### TODOS
+
+- [ ] Features
+  - [ ] Enable operator remote debugging
+- [ ] CRD validation
+  - [ ] Admission webhook ? https://github.com/operator-framework/josdk-webhooks
+- [ ] Operator release
+  - [ ] Publish operator image as part of release process
+  - [ ] Publish operator Helm charts
+  - [ ] Publish manifests + Kustomize?
+  - [ ] Operator Lifecycle Manager (OLM) support
+    - [ ] Generate and Validate ClusterServiceVersion (CSV) metadata
+      - https://olm.operatorframework.io/docs/tasks/creating-operator-manifests/
+      - https://github.com/operator-framework/operator-lifecycle-manager/blob/master/doc/design/building-your-csv.md
+      - https://k8s-operatorhub.github.io/community-operators/packaging-required-fields/
+    - [ ] Publish bundle image as part of release process
+    - [ ] Publish bundle to operatorhub.io:
+      - How to contribute: https://operatorhub.io/contribute
+      - Instructions for packaging: https://k8s-operatorhub.github.io/community-operators/packaging-operator/
+      - Testing the bundle locally: https://k8s-operatorhub.github.io/community-operators/testing-operators/
+    - [ ] Publish a certified bundle to Openshift OperatorHub?
+      - https://github.com/redhat-openshift-ecosystem/community-operators-prod
+      - https://github.com/redhat-openshift-ecosystem/certified-operators
+- [ ] Tests
+  - [ ] Unit tests
+    - [ ] Smoke test
+    - [ ] Missing / invalid configs
+    - [ ] Out-of-cluster tests ?
+  - [ ] Integration tests: 
+    - [ ] All version stores
+    - [ ] https://github.com/java-operator-sdk/jenvtest ?
+    - Examples: https://github.com/operator-framework/java-operator-sdk/tree/main/operator-framework/src/test/java/io/javaoperatorsdk/operator
+  - [ ] E2E tests / Adhoc tests
+    - [ ] Deploy operator in a real cluster (GCP / EKS / AKS / OCP)
+- [ ] GC
+  - [ ] Add a NessieTask CRD ? Reuse Job CRD? 
+  - [ ] Automatically set up a temporary database for Nessie GC
+  - [ ] Periodic Nessie GC
+
+### Known issues
+
+- [ ] Check generated RBAC rules
+
+### Prerequisites
+
+- Operator SDK: https://sdk.operatorframework.io/docs/installation/
+
+### Building the operator image
+
+```bash
+make docker-build
+```
+
+### Adhoc testing with Minikube
+
+Install [minikube](https://minikube.sigs.k8s.io/docs/start/).
+
+If you need ingress, install the ingress addon:
+
+```bash
+minikube addons enable ingress
+minikube tunnel
+```
+
+Create the nessie-operator and nessie-ns namespaces (only needed once):
+
+```bash
+kubectl create namespace nessie-operator
+kubectl create namespace nessie-ns
+```
+
+Grant admin rights to the nessie-operator service account (only needed once):
+
+```bash
+kubectl apply -f examples/nessie-operator-rbac.yaml
+```
+
+Build the operator docker image _inside_ minikube to facilitate testing (doesn't need a registry):
+
+```bash
+eval $(minikube docker-env)
+make docker-build PULL_POLICY=IfNotPresent
+```
+
+Note: the `PULL_POLICY=IfNotPresent` is required to avoid pulling the image from a registry.
+
+Install the CRDs and deploy the operator in the nessie-operator namespace:
+
+```bash
+make install deploy
+```
+
+Create a Nessie resource in the nessie-ns namespace:
+
+```bash
+kubectl apply -n nessie-ns -f examples/nessie-simple.yaml
+```
+
+You should see 1 pod, 1 deployment and 1 service running.
+
+### Testing OLM bundles (WIP)
+
+Install OLM in your cluster:
+
+```bash
+operator-sdk olm install
+```
+
+Manual bundle deployment instructions:
+https://docs.quarkiverse.io/quarkus-operator-sdk/dev/index.html#_deployment
+
+Install `opm` (operator package manager): download the binary from:
+https://github.com/operator-framework/operator-registry/releases/tag/v1.36.0
+
+MacOS users will need to remove the quarantine attribute from the binary:
+
+```bash
+xattr -d com.apple.quarantine ~/bin/opm
+``` 

operator/build.gradle.kts

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.tools.ant.filters.ReplaceTokens
+
+plugins {
+  alias(libs.plugins.quarkus)
+  id("nessie-conventions-quarkus")
+  id("nessie-jacoco")
+}
+
+extra["maven.name"] = "Nessie - Kubernetes Operator"
+
+dependencies {
+  implementation(enforcedPlatform(libs.quarkus.bom))
+  implementation(enforcedPlatform(libs.quarkus.operator.sdk.bom))
+
+  implementation("io.quarkiverse.operatorsdk:quarkus-operator-sdk")
+  implementation("io.quarkiverse.operatorsdk:quarkus-operator-sdk-bundle-generator")
+  implementation("io.quarkus:quarkus-micrometer-registry-prometheus")
+  implementation("io.quarkus:quarkus-container-image-jib")
+
+  compileOnly("io.sundr:builder-annotations:0.103.0")
+  compileOnly("io.fabric8:generator-annotations")
+
+  annotationProcessor(enforcedPlatform(libs.quarkus.bom))
+  annotationProcessor("io.sundr:builder-annotations:0.103.0")
+  // see https://github.com/sundrio/sundrio/issues/104
+  annotationProcessor("io.fabric8:kubernetes-client")
+
+  testImplementation(enforcedPlatform(libs.quarkus.bom))
+  testImplementation("io.quarkus:quarkus-junit5")
+  testImplementation("io.quarkus:quarkus-junit5-mockito")
+  testImplementation("io.quarkus:quarkus-test-kubernetes-client")
+  testImplementation("org.bouncycastle:bcpkix-jdk18on")
+  testImplementation(platform(libs.junit.bom))
+  testImplementation(libs.bundles.junit.testing)
+  testImplementation(libs.awaitility)
+}
+
+listOf("javadoc", "sourcesJar").forEach { name ->
+  tasks.named(name).configure { dependsOn(tasks.named("compileQuarkusGeneratedSourcesJava")) }
+}
+
+listOf("checkstyleTest", "compileTestJava").forEach { name ->
+  tasks.named(name).configure { dependsOn(tasks.named("compileQuarkusTestGeneratedSourcesJava")) }
+}
+
+tasks.named<ProcessResources>("processTestResources").configure {
+  inputs.property("projectVersion", project.version)
+  filter(ReplaceTokens::class, mapOf("tokens" to mapOf("projectVersion" to project.version)))
+}

operator/examples/nessie-autoscaling.yaml

@@ -0,0 +1,19 @@
+apiVersion: nessie.projectnessie.org/v1alpha1
+kind: Nessie
+metadata:
+  name: nessie-autoscaling
+spec:
+  size: 1
+  logLevel: INFO
+  image:
+    repository: projectnessie/nessie
+    tag: 0.75.0
+  versionStore:
+    type: Jdbc
+    jdbc:
+      jdbcUrl: jdbc:h2:mem:nessie
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 3
+    targetCpuUtilizationPercentage: 50