feat: make generator concurrency configurable

nestoca · Aug 30, 2024 · c65962c · c65962c
1 parent 581b620
commit c65962c
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 6 deletions.
diff --git a/cmd/server/api_get_params_test.go b/cmd/server/api_get_params_test.go
@@ -74,6 +74,7 @@ func TestGetParamsE2E(t *testing.T) {
 			LoadJoyContext: generator.RepoLoader(repo),
 			Logger:         logger,
 			ChartPuller:    generator.MakeChartPuller(logger),
+			Concurrency:    4,
 		},
 	})
 

diff --git a/cmd/server/config.go b/cmd/server/config.go
@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"time"
 
 	"github.com/davidmdm/conf"
@@ -21,6 +22,10 @@ type Config struct {
 
 	CacheRoot string
 
+	Generator struct {
+		Concurrency int
+	}
+
 	Google struct {
 		Repository          string
 		CredentialsFilePath string
@@ -70,6 +75,8 @@ func GetConfig() Config {
 	conf.Var(conf.Environ, &cfg.Otel.ServiceName, "OTEL_SERVICE_NAME", conf.Default("joy-generator"))
 	conf.Var(conf.Environ, &cfg.Otel.ServiceVersion, "OTEL_SERVICE_VERSION")
 
+	conf.Var(conf.Environ, &cfg.Generator.Concurrency, "GENERATOR_CONCURRENCY", conf.Default(runtime.NumCPU()))
+
 	conf.Environ.MustParse()
 
 	if path := cfg.Google.CredentialsFilePath; path != "" {

diff --git a/cmd/server/handler.go b/cmd/server/handler.go
@@ -149,7 +149,7 @@ func SpanNamer(c *gin.Context) {
 	if span := trace.SpanFromContext(c.Request.Context()); span.IsRecording() {
 		name := c.Request.Method + cmp.Or(c.FullPath(), "route_not_found")
 		name = nonWordCharacters.ReplaceAllString(name, "_")
-    name = strings.ToLower(name)
+		name = strings.ToLower(name)
 		span.SetName(name)
 	}
 }
diff --git a/internal/generator/generator.go b/internal/generator/generator.go
@@ -26,6 +26,7 @@ type Generator struct {
 	LoadJoyContext JoyLoaderFunc
 	Logger         zerolog.Logger
 	ChartPuller    helm.Puller
+	Concurrency    int
 }
 
 type MutexMap sync.Map
@@ -159,13 +160,13 @@ func (generator *Generator) Run(ctx context.Context) ([]Result, error) {
 	var (
 		wg                 sync.WaitGroup
 		reconciledReleases = make([]Result, len(releases))
-		semaphore          = make(chan struct{}, 8)
+		semaphore          = make(chan struct{}, min(generator.Concurrency, 1))
 	)
 
 	for i, release := range releases {
 		semaphore <- struct{}{}
 
-    wg.Add(1)
+		wg.Add(1)
 
 		go func() {
 			defer wg.Done()

diff --git a/internal/generator/generator_test.go b/internal/generator/generator_test.go
@@ -115,13 +115,13 @@ func TestGenerator(t *testing.T) {
 			DefaultChartRef: "custom",
 			Charts: func() map[string]helm.Chart {
 				charts := maps.Clone(baseCharts)
-        custom := charts["custom"]
+				custom := charts["custom"]
 				custom.Mappings = map[string]any{
 					"annotations.test": true,
 					"image":            "image@{{ .Release.Spec.Version }}",
 				}
-        charts["custom"] = custom
-        return charts
+				charts["custom"] = custom
+				return charts
 			}(),
 			ExpectedRelease: &v1alpha1.Release{
 				ApiVersion:      "joy.nesto.ca/v1alpha1",