Add Control Loop

.gitignore

@@ -18,4 +18,5 @@
 /nats-boot-config
 /nats-boot-config.docker
 /tools
+/bin
 /.idea

Makefile

@@ -1,5 +1,9 @@
 export GO111MODULE := on
 
+SHELL=/usr/bin/env bash
+
+ENVTEST_K8S_VERSION = 1.31.0
+
 now := $(shell date -u +%Y-%m-%dT%H:%M:%S%z)
 gitBranch := $(shell git rev-parse --abbrev-ref HEAD)
 gitCommit := $(shell git rev-parse --short HEAD)
@@ -9,8 +13,7 @@ VERSION ?= version-not-set
 linkerVars := -X main.BuildTime=$(now) -X main.GitInfo=$(gitBranch)-$(gitCommit)$(repoDirty) -X main.Version=$(VERSION)
 drepo ?= natsio
 
-jetstreamGenIn:= $(shell grep -l -R -F "// +k8s:" pkg/jetstream/apis)
-jetstreamSrc := $(shell find cmd/jetstream-controller pkg/jetstream controllers/jetstream -name "*.go") pkg/jetstream/apis/jetstream/v1beta2/zz_generated.deepcopy.go
+jetstreamSrc := $(shell find cmd/jetstream-controller pkg/jetstream internal/controller controllers/jetstream -name "*.go") pkg/jetstream/apis/jetstream/v1beta2/zz_generated.deepcopy.go
 
 configReloaderSrc := $(shell find cmd/nats-server-config-reloader/ pkg/natsreloader/ -name "*.go")
 
@@ -27,20 +30,21 @@ default:
 	#   make nats-server-config-reloader
 	#   make nats-boot-config
 
-pkg/jetstream/generated pkg/jetstream/apis/jetstream/v1beta2/zz_generated.deepcopy.go: fetch-modules $(jetstreamGenIn) pkg/k8scodegen/file-header.txt
+generate: fetch-modules pkg/k8scodegen/file-header.txt
 	rm -rf pkg/jetstream/generated
-	# Temporary chmod fix until we migrate to kube_codegen.sh
 	D="$(codeGeneratorDir)"; : "$${D:=`go list -m -f '{{.Dir}}' k8s.io/code-generator`}"; \
-	chmod u+x "$$D/generate-internal-groups.sh"; \
-	GOFLAGS='' bash "$$D/generate-groups.sh" all \
-		github.com/nats-io/nack/pkg/jetstream/generated \
-		github.com/nats-io/nack/pkg/jetstream/apis \
-		"jetstream:v1beta2" \
-		--output-base . \
-		--go-header-file pkg/k8scodegen/file-header.txt
-	mv github.com/nats-io/nack/pkg/jetstream/generated pkg/jetstream/generated
-	mv github.com/nats-io/nack/pkg/jetstream/apis/jetstream/v1beta2/zz_generated.deepcopy.go pkg/jetstream/apis/jetstream/v1beta2/zz_generated.deepcopy.go
-	rm -rf github.com
+	source "$$D/kube_codegen.sh" ; \
+	kube::codegen::gen_helpers \
+	  --boilerplate pkg/k8scodegen/file-header.txt \
+	  pkg/jetstream/apis; \
+	kube::codegen::gen_client \
+		--with-watch \
+		--with-applyconfig \
+		--boilerplate pkg/k8scodegen/file-header.txt \
+		--output-dir pkg/jetstream/generated \
+		--output-pkg github.com/nats-io/nack/pkg/jetstream/generated \
+		--one-input-api jetstream/v1beta2 \
+		pkg/jetstream/apis
 
 jetstream-controller: $(jetstreamSrc)
 	go build -race -o $@ \
@@ -173,10 +177,39 @@ fetch-modules:
 .PHONY: build
 build: jetstream-controller nats-server-config-reloader nats-boot-config
 
+# Setup envtest tools based on a operator-sdk project makefile
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
+# $1 - target path with name of binary (ideally with version)
+# $2 - package url which can be installed
+# $3 - specific version of package
+define go-install-tool
+@[ -f $(1) ] || { \
+set -e; \
+package=$(2)@$(3) ;\
+echo "Downloading $${package}" ;\
+GOBIN=$(LOCALBIN) go install $${package} ;\
+mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\
+}
+endef
+
+ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION)
+ENVTEST_VERSION ?= release-0.19
+
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION))
+
+
 .PHONY: test
-test:
-	go vet ./controllers/... ./pkg/natsreloader/...
-	go test -race -cover -count=1 -timeout 10s ./controllers/... ./pkg/natsreloader/...
+test: envtest
+	go vet ./controllers/... ./pkg/natsreloader/... ./internal/controller/...
+	$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path ## Get k8s binaries
+	go test -race -cover -count=1 -timeout 10s ./controllers/... ./pkg/natsreloader/... ./internal/controller/...
 
 .PHONY: clean
 clean:

README.md

@@ -9,22 +9,30 @@
 
 ## JetStream Controller
 
-The JetStream controllers allows you to manage [NATS JetStream](https://github.com/nats-io/jetstream) [Streams](https://github.com/nats-io/jetstream#streams-1) and [Consumers](https://github.com/nats-io/jetstream#consumers-1) via K8S CRDs.
+The JetStream controllers allows you to manage [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) [Streams](https://docs.nats.io/nats-concepts/jetstream/streams), [Consumers](https://docs.nats.io/nats-concepts/jetstream/consumers), [Key/Value Stores](https://docs.nats.io/nats-concepts/jetstream/key-value-store), and [Object Stores](https://docs.nats.io/nats-concepts/jetstream/obj_store) via Kubernetes CRDs.
+
+Resources managed by NACK controllers are expected to _exclusively_ be managed by NACK, and configuration state will be enforced if mutated by an external client.
+
+## [API Reference](docs/api.md)
 
 ### Getting started
 
-First install the JetStream CRDs:
+Install with Helm:
 
-```sh
-$ kubectl apply -f https://github.com/nats-io/nack/releases/latest/download/crds.yml
 ```
+helm repo add nats https://nats-io.github.io/k8s/helm/charts/
+helm upgrade --install nats nats/nats --set config.jetstream.enabled=true --set config.cluster.enabled=true
+helm upgrade --install nack nats/nack --set jetstream.nats.url=nats://nats.default.svc.cluster.local:4222
+```
+
+#### (Optional) Enable Experimental `controller-runtime` Controllers
 
-Now install with Helm:
+> **Note**: The updated controllers will more reliably enforce resource state. If migrating from an older version of NACK, as long as all NATS resources are in-sync with NACK resources no modifications are expected.
+>
+> The `jetstream-controller` logs will contain a diff of any changes the controller has made.
 
 ```
-helm repo add nats https://nats-io.github.io/k8s/helm/charts/
-helm install nats nats/nats --set=config.jetstream.enabled=true
-helm install nack nats/nack --set jetstream.nats.url=nats://nats:4222
+helm upgrade nack nats/nack --set jetstream.additionalArgs={--control-loop=true}
 ```
 
 #### Creating Streams and Consumers
@@ -66,6 +74,28 @@ spec:
   filterSubject: orders.received
   maxDeliver: 20
   ackPolicy: explicit
+---
+apiVersion: jetstream.nats.io/v1beta2
+kind: KeyValue
+metadata:
+  name: my-key-value
+spec:
+  bucket: my-key-value
+  history: 20
+  storage: file
+  maxBytes: 2048
+  compression: true
+---
+apiVersion: jetstream.nats.io/v1beta2
+kind: ObjectStore
+metadata:
+  name: my-object-store
+spec:
+  bucket: my-object-store
+  storage: file
+  replicas: 1
+  maxBytes: 536870912 # 512 MB
+  compression: true
 ```
 
 ```sh
@@ -75,7 +105,7 @@ $ kubectl apply -f https://raw.githubusercontent.com/nats-io/nack/main/deploy/ex
 # Check if it was successfully created.
 $ kubectl get streams
 NAME       STATE     STREAM NAME   SUBJECTS
-mystream   Created   mystream      [orders.*]
+mystream   Ready     mystream      [orders.*]
 
 # Create a push-based consumer
 $ kubectl apply -f https://raw.githubusercontent.com/nats-io/nack/main/deploy/examples/consumer_push.yml
@@ -86,8 +116,8 @@ $ kubectl apply -f https://raw.githubusercontent.com/nats-io/nack/main/deploy/ex
 # Check if they were successfully created.
 $ kubectl get consumers
 NAME               STATE     STREAM     CONSUMER           ACK POLICY
-my-pull-consumer   Created   mystream   my-pull-consumer   explicit
-my-push-consumer   Created   mystream   my-push-consumer   none
+my-pull-consumer   Ready     mystream   my-pull-consumer   explicit
+my-push-consumer   Ready     mystream   my-push-consumer   none
 
 # If you end up in an Errored state, run kubectl describe for more info.
 #     kubectl describe streams mystream
@@ -178,7 +208,7 @@ metadata:
 spec:
   name: a
   servers:
-  - nats://nats:4222
+    - nats://nats:4222
   tls:
     secret:
       name: nack-a-tls
@@ -209,31 +239,32 @@ Server URL and TLS certificates.
 
 ```sh
 # Install cert-manager
-kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.0/cert-manager.yaml
+kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.17.0/cert-manager.yaml
 
 # Install TLS certs
 cd examples/secure
+
 # Install certificate issuer
 kubectl apply -f issuer.yaml
+
 # Install account A cert
 kubectl apply -f nack-a-client-tls.yaml
+
 # Install server cert
 kubectl apply -f server-tls.yaml
+
 # Install nats-box cert
 kubectl apply -f client-tls.yaml
 
 # Install NATS cluster
-helm install -f nats-helm.yaml nats nats/nats
+helm upgrade --install -f nats-helm.yaml nats nats/nats
+
 # Verify pods are healthy
 kubectl get pods
 
-# Install nats-box to run nats cli later
-kubectl apply -f nats-client-box.yaml
-
 # Install JetStream Controller from nack
-helm install --set jetstream.enabled=true jetstream-controller nats/nack
-# Install CRDs
-kubectl apply -f ../../deploy/crds.yml
+helm upgrade --install nack nats/nack --set jetstream.enabled=true
+
 # Verify pods are healthy
 kubectl get pods
 
@@ -242,6 +273,7 @@ kubectl apply -f nack/nats-account-a.yaml
 
 # Create stream using account A
 kubectl apply -f nack/nats-stream-foo-a.yaml
+
 # Create consumer using account A
 kubectl apply -f nack/nats-consumer-bar-a.yaml
 ```
@@ -251,30 +283,28 @@ container to run the management CLI.
 
 ```sh
 # Get container shell
-kubectl exec -it nats-client-box-abc-123 -- sh
-# Change to TLS directory
-cd /etc/nats-certs/clients/nack-a-tls
+kubectl exec -it deployment/nats-box -- /bin/sh
 ```
 
 There should now be some Streams available, verify with `nats` command.
 
 ```sh
 # List streams
-nats --tlscert tls.crt --tlskey tls.key --tlsca ca.crt -s tls://nats.default.svc.cluster.local stream ls
+nats stream ls
 ```
 
 You can now publish messages on a Stream.
 
 ```sh
 # Push message
-nats --tlscert tls.crt --tlskey tls.key --tlsca ca.crt -s tls://nats.default.svc.cluster.local pub foo hi
+nats pub foo hi
 ```
 
 And pull messages from a Consumer.
 
 ```sh
 # Pull message
-nats --tlscert tls.crt --tlskey tls.key --tlsca ca.crt -s tls://nats.default.svc.cluster.local consumer next foo bar
+nats consumer next foo bar
 ```
 
 ### Local Development
@@ -298,6 +328,7 @@ nats-server -DV -js
 ```
 
 Build Docker image
+
 ```sh
 make jetstream-controller-docker ver=1.2.3
 ```
@@ -314,15 +345,15 @@ For more information see the
 
 ```
 helm repo add nats https://nats-io.github.io/k8s/helm/charts/
-helm install my-nats nats/nats
+helm upgrade --install nats nats/nats
 ```
 
 ### Configuring
 
 ```yaml
 reloader:
   enabled: true
-  image: natsio/nats-server-config-reloader:0.6.0
+  image: natsio/nats-server-config-reloader:0.16.1
   pullPolicy: IfNotPresent
 ```
 
@@ -337,6 +368,7 @@ make nats-server-config-reloader
 ```
 
 Build Docker image
+
 ```sh
 make nats-server-config-reloader-docker ver=1.2.3
 ```
@@ -350,14 +382,14 @@ For more information see the
 
 ```
 helm repo add nats https://nats-io.github.io/k8s/helm/charts/
-helm install my-nats nats/nats
+helm upgrade --install nats nats/nats
 ```
 
 ### Configuring
 
 ```yaml
 bootconfig:
-  image: natsio/nats-boot-config:0.5.2
+  image: natsio/nats-boot-config:0.16.1
   pullPolicy: IfNotPresent
 ```
 
@@ -372,6 +404,7 @@ make nats-boot-config
 ```
 
 Build Docker image
+
 ```sh
 make nats-boot-config-docker ver=1.2.3
 ```

cicd/Dockerfile

@@ -1,7 +1,7 @@
-#syntax=docker/dockerfile-upstream:1.5
+#syntax=docker/dockerfile:1.13
 ARG GO_APP
 
-FROM alpine:3.21.2 as deps
+FROM alpine:3.21.3 as deps
 
 ARG GO_APP
 ARG GORELEASER_DIST_DIR=/go/src/dist
@@ -28,7 +28,7 @@ RUN <<EOT
   cp ${BIN_PATH} /go/bin
 EOT
 
-FROM alpine:3.21.2
+FROM alpine:3.21.3
 
 ARG GO_APP
 ENV GO_APP ${GO_APP}

cicd/Dockerfile_goreleaser

@@ -1,6 +1,7 @@
-#syntax=docker/dockerfile-upstream:1.5
+#syntax=docker/dockerfile:1.13
 FROM --platform=$BUILDPLATFORM golang:1.24.0-bullseye as build
 
+
 RUN <<EOT
     set -e