feat(payments): v3 implementation

.gitignore

@@ -1,4 +1,5 @@
 .idea
+.vscode
 openapi/build/generate.json-e
 openapi/build/generate.json.bak
 coverage.out

components/operator/go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect

components/operator/go.sum

@@ -34,12 +34,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -166,8 +164,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

components/operator/internal/resources/payments/deployments.go

@@ -1,10 +1,15 @@
 package payments
 
 import (
+	"fmt"
+	"strings"
+
+	"github.com/formancehq/go-libs/pointer"
 	"github.com/formancehq/operator/internal/resources/brokers"
 	"github.com/formancehq/operator/internal/resources/brokertopics"
 	"github.com/formancehq/operator/internal/resources/caddy"
 	"github.com/formancehq/operator/internal/resources/registries"
+	"github.com/formancehq/operator/internal/resources/resourcereferences"
 	"github.com/formancehq/operator/internal/resources/settings"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,6 +33,58 @@ func getEncryptionKey(ctx core.Context, payments *v1beta1.Payments) (string, err
 	return "", nil
 }
 
+func temporalEnvVars(ctx core.Context, stack *v1beta1.Stack, payments *v1beta1.Payments) ([]v1.EnvVar, error) {
+
+	temporalURI, err := settings.RequireURL(ctx, stack.Name, "temporal", "dsn")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := validateTemporalURI(temporalURI); err != nil {
+		return nil, err
+	}
+
+	if secret := temporalURI.Query().Get("secret"); secret != "" {
+		_, err = resourcereferences.Create(ctx, payments, "payments-temporal", secret, &v1.Secret{})
+	} else {
+		err = resourcereferences.Delete(ctx, payments, "payments-temporal")
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	env := make([]v1.EnvVar, 0)
+	env = append(env,
+		core.Env("TEMPORAL_TASK_QUEUE", stack.Name),
+		core.Env("TEMPORAL_ADDRESS", temporalURI.Host),
+		core.Env("TEMPORAL_NAMESPACE", temporalURI.Path[1:]),
+	)
+
+	if secret := temporalURI.Query().Get("secret"); secret == "" {
+		temporalTLSCrt, err := settings.GetStringOrEmpty(ctx, stack.Name, "temporal", "tls", "crt")
+		if err != nil {
+			return nil, err
+		}
+
+		temporalTLSKey, err := settings.GetStringOrEmpty(ctx, stack.Name, "temporal", "tls", "key")
+		if err != nil {
+			return nil, err
+		}
+
+		env = append(env,
+			core.Env("TEMPORAL_SSL_CLIENT_KEY", temporalTLSKey),
+			core.Env("TEMPORAL_SSL_CLIENT_CERT", temporalTLSCrt),
+		)
+	} else {
+		env = append(env,
+			core.EnvFromSecret("TEMPORAL_SSL_CLIENT_KEY", secret, "tls.key"),
+			core.EnvFromSecret("TEMPORAL_SSL_CLIENT_CERT", secret, "tls.crt"),
+		)
+	}
+
+	return env, nil
+}
+
 func commonEnvVars(ctx core.Context, stack *v1beta1.Stack, payments *v1beta1.Payments, database *v1beta1.Database) ([]v1.EnvVar, error) {
 	env := make([]v1.EnvVar, 0)
 	otlpEnv, err := settings.GetOTELEnvVars(ctx, stack.Name, core.LowerCamelCaseKind(ctx, payments))
@@ -57,13 +114,15 @@ func commonEnvVars(ctx core.Context, stack *v1beta1.Stack, payments *v1beta1.Pay
 	env = append(env,
 		core.Env("POSTGRES_DATABASE_NAME", "$(POSTGRES_DATABASE)"),
 		core.Env("CONFIG_ENCRYPTION_KEY", encryptionKey),
+		core.Env("PLUGIN_DIRECTORY_PATH", "/plugins"),
+		core.Env("PLUGIN_MAGIC_COOKIE", "magic-value"), // TODO(polo): change value
 	)
 
 	return env, nil
 }
 
 func createFullDeployment(ctx core.Context, stack *v1beta1.Stack,
-	payments *v1beta1.Payments, database *v1beta1.Database, image string) error {
+	payments *v1beta1.Payments, database *v1beta1.Database, image string, v3 bool) error {
 
 	env, err := commonEnvVars(ctx, stack, payments, database)
 	if err != nil {
@@ -101,11 +160,25 @@ func createFullDeployment(ctx core.Context, stack *v1beta1.Stack,
 		env = append(env, brokers.GetPublisherEnvVars(stack, broker, "payments", "")...)
 	}
 
+	if v3 {
+		temporalEnvVars, err := temporalEnvVars(ctx, stack, payments)
+		if err != nil {
+			return err
+		}
+
+		env = append(env, temporalEnvVars...)
+	}
+
 	serviceAccountName, err := settings.GetAWSServiceAccount(ctx, stack.Name)
 	if err != nil {
 		return err
 	}
 
+	appOpts := applications.WithProbePath("/_health")
+	if v3 {
+		appOpts = applications.WithProbePath("/_healthcheck")
+	}
+
 	err = applications.
 		New(payments, &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
@@ -120,8 +193,12 @@ func createFullDeployment(ctx core.Context, stack *v1beta1.Stack,
 							Args:          []string{"serve"},
 							Env:           env,
 							Image:         image,
-							LivenessProbe: applications.DefaultLiveness("http", applications.WithProbePath("/_health")),
+							LivenessProbe: applications.DefaultLiveness("http", appOpts),
 							Ports:         []v1.ContainerPort{applications.StandardHTTPPort()},
+							// TODO(polo): only for v3
+							SecurityContext: &v1.SecurityContext{
+								ReadOnlyRootFilesystem: pointer.For(false),
+							},
 						}},
 						// Ensure empty
 						InitContainers: []v1.Container{},
@@ -298,3 +375,19 @@ func createGateway(ctx core.Context, stack *v1beta1.Stack, p *v1beta1.Payments)
 		New(p, deploymentTemplate).
 		Install(ctx)
 }
+
+func validateTemporalURI(temporalURI *v1beta1.URI) error {
+	if temporalURI.Scheme != "temporal" {
+		return fmt.Errorf("invalid temporal uri: %s", temporalURI.String())
+	}
+
+	if temporalURI.Path == "" {
+		return fmt.Errorf("invalid temporal uri: %s", temporalURI.String())
+	}
+
+	if !strings.HasPrefix(temporalURI.Path, "/") {
+		return fmt.Errorf("invalid temporal uri: %s", temporalURI.String())
+	}
+
+	return nil
+}

components/operator/internal/resources/payments/init.go

@@ -92,11 +92,13 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, p *v1beta1.Payments, version s
 		}
 	}
 
-	if semver.IsValid(version) && semver.Compare(version, "v1.0.0-alpha") < 0 {
-		if err := createFullDeployment(ctx, stack, p, database, image); err != nil {
+	switch {
+	case semver.IsValid(version) && semver.Compare(version, "v1.0.0-alpha") < 0:
+		if err := createFullDeployment(ctx, stack, p, database, image, false); err != nil {
 			return err
 		}
-	} else {
+	case semver.IsValid(version) && semver.Compare(version, "v1.0.0-alpha") >= 0 &&
+		semver.Compare(version, "v3.0.0") < 0:
 		if err := createReadDeployment(ctx, stack, p, database, image); err != nil {
 			return err
 		}
@@ -107,6 +109,10 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, p *v1beta1.Payments, version s
 		if err := createGateway(ctx, stack, p); err != nil {
 			return err
 		}
+	case !semver.IsValid(version) || semver.Compare(version, "v3.0.0") >= 0:
+		if err := createFullDeployment(ctx, stack, p, database, image, true); err != nil {
+			return err
+		}
 	}
 
 	if err := benthosstreams.LoadFromFileSystem(ctx, benthos.Streams, p, "streams/payments", "ingestion"); err != nil {

components/operator/internal/tests/payments_controller_test.go

@@ -10,16 +10,16 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var _ = Describe("PaymentsController", func() {
 	Context("When creating a Payments object", func() {
 		var (
-			stack            *v1beta1.Stack
-			payments         *v1beta1.Payments
-			databaseSettings *v1beta1.Settings
+			stack               *v1beta1.Stack
+			payments            *v1beta1.Payments
+			databaseSettings    *v1beta1.Settings
+			temporalDSNSettings *v1beta1.Settings
 		)
 		BeforeEach(func() {
 			stack = &v1beta1.Stack{
@@ -35,10 +35,12 @@ var _ = Describe("PaymentsController", func() {
 				},
 			}
 			databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name)
+			temporalDSNSettings = settings.New(uuid.NewString(), "temporal.dsn", "temporal://localhost/namespace", stack.Name)
 		})
 		JustBeforeEach(func() {
 			Expect(Create(stack)).To(Succeed())
 			Expect(Create(databaseSettings)).To(Succeed())
+			Expect(Create(temporalDSNSettings)).To(Succeed())
 			Expect(Create(payments)).To(Succeed())
 		})
 		AfterEach(func() {
@@ -55,45 +57,13 @@ var _ = Describe("PaymentsController", func() {
 					return reference
 				}).Should(BeTrue())
 			})
-			By("Should create a read deployment with a service", func() {
-				deployment := &appsv1.Deployment{}
-				Eventually(func() error {
-					return LoadResource(stack.Name, "payments-read", deployment)
-				}).Should(Succeed())
-				Expect(deployment).To(BeControlledBy(payments))
-
-				service := &corev1.Service{}
-				Eventually(func() error {
-					return LoadResource(stack.Name, "payments-read", service)
-				}).Should(Succeed())
-				Expect(service).To(BeControlledBy(payments))
-			})
-			By("Should create a connectors deployment with a service", func() {
-				deployment := &appsv1.Deployment{}
-				Eventually(func() error {
-					return LoadResource(stack.Name, "payments-connectors", deployment)
-				}).Should(Succeed())
-				Expect(deployment).To(BeControlledBy(payments))
-
-				service := &corev1.Service{}
-				Eventually(func() error {
-					return LoadResource(stack.Name, "payments-connectors", service)
-				}).Should(Succeed())
-				Expect(service).To(BeControlledBy(payments))
-			})
-			By("Should create a gateway", func() {
+			By("Should create a deployment", func() {
 				deployment := &appsv1.Deployment{}
 				Eventually(func() error {
 					return LoadResource(stack.Name, "payments", deployment)
 				}).Should(Succeed())
 				Expect(deployment).To(BeControlledBy(payments))
 			})
-			By("Should create a new GatewayHTTPAPI object", func() {
-				httpService := &v1beta1.GatewayHTTPAPI{}
-				Eventually(func() error {
-					return LoadResource("", core.GetObjectName(stack.Name, "payments"), httpService)
-				}).Should(Succeed())
-			})
 		})
 		Context("With Search enabled", func() {
 			var search *v1beta1.Search

components/operator/tools/kubectl-stacks/go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect

components/operator/tools/kubectl-stacks/go.sum

@@ -73,9 +73,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
@@ -86,7 +85,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -316,8 +314,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

components/operator/tools/utils/go.mod

@@ -5,7 +5,7 @@ go 1.22
 toolchain go1.22.6
 
 require (
-	github.com/formancehq/go-libs v1.5.0
+	github.com/formancehq/go-libs v1.6.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.1
 )
@@ -34,7 +34,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
@@ -70,10 +70,10 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
-	google.golang.org/grpc v1.64.1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect
+	google.golang.org/grpc v1.66.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 )