From c2c0a0a363b9e25d8f6b07a40823d3f5443e0833 Mon Sep 17 00:00:00 2001 From: David Desmarais-Michaud Date: Wed, 8 May 2024 13:58:38 -0400 Subject: [PATCH] feat(PL-2701): update joy to unify schema values --- Dockerfile | 4 +- chart/templates/deployment.yaml | 13 ++ chart/templates/secret.yaml | 15 ++- .../with-google-credentials/expected.yaml | 124 ++++++++++++++++++ .../tests/with-google-credentials/values.yaml | 20 +++ cmd/server/api_get_params_test.go | 25 +++- cmd/server/config.go | 26 ++++ cmd/server/main.go | 33 ++++- go.mod | 5 +- go.sum | 34 ++++- internal/generator/generator.go | 34 ++++- internal/generator/generator_test.go | 6 +- 12 files changed, 317 insertions(+), 22 deletions(-) create mode 100644 chart/tests/with-google-credentials/expected.yaml create mode 100644 chart/tests/with-google-credentials/values.yaml diff --git a/Dockerfile b/Dockerfile index 2080715..ca690f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,12 +25,14 @@ COPY --chown=golang:root internal ./internal RUN go build -v -o joy-generator ./cmd/server -FROM scratch AS prod +FROM alpine AS prod COPY --from=build /etc/passwd /etc/group /etc/ COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=build --chown=golang:root /app/joy-generator /app/ +RUN apk add helm + USER golang:root EXPOSE 8080 diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 8e0228c..495abed 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -34,6 +34,11 @@ spec: secret: secretName: {{ include "joy-generator.fullname" . }}-github-app-key {{- end }} + {{- if .Values.googleCredentials }} + - name: google-credentials + secret: + secretName: {{ include "joy-generator.fullname" .}}-google-credentials + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -47,6 +52,10 @@ spec: - mountPath: /etc/joy/config name: github-app-private-key {{- end }} + {{- if .Values.googleCredentials }} + - mountPath: /etc/joy/secrets + name: google-credentials + {{- end }} env: {{- range $key, $value := .Values.env }} {{- if $value }} @@ -58,6 +67,10 @@ spec: - name: GH_APP_PRIVATE_KEY_PATH value: /etc/joy/config/githubApp.pem {{- end }} + {{- if .Values.googleCredentials }} + - name: GOOGLE_CREDENTIALS_FILE + value: /etc/joy/secrets/credentials.json + {{- end }} envFrom: - secretRef: name: {{ include "joy-generator.fullname" . }}-config diff --git a/chart/templates/secret.yaml b/chart/templates/secret.yaml index 90fbee5..872da16 100644 --- a/chart/templates/secret.yaml +++ b/chart/templates/secret.yaml @@ -25,9 +25,8 @@ stringData: {{ toYaml .Values.secretEnv.values | indent 2 }} {{- end }} ---- - {{- if .Values.env.GH_APP_ID }} +--- {{- and (required ".Values.env.GH_APP_INSTALLATION_ID is required" .Values.env.GH_APP_INSTALLATION_ID) "" | -}} {{- and (required ".Values.githubAppPrivateKey is required" .Values.githubAppPrivateKey) "" -}} {{- if $sealedSecret }} @@ -54,3 +53,15 @@ stringData: {{ .Values.githubAppPrivateKey | indent 4 }} {{- end }} {{- end }} + + +{{- if .Values.googleCredentials }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "joy-generator.fullname" . }}-google-credentials +stringData: + credentials.json: | +{{ .Values.googleCredentials | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/chart/tests/with-google-credentials/expected.yaml b/chart/tests/with-google-credentials/expected.yaml new file mode 100644 index 0000000..bcc1c99 --- /dev/null +++ b/chart/tests/with-google-credentials/expected.yaml @@ -0,0 +1,124 @@ +--- +# Source: joy-generator/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-release-joy-generator-config +stringData: + GH_TOKEN: "12312312312312" + PLUGIN_TOKEN: '@very!l0ngands3curet0ken' +--- +# Source: joy-generator/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-release-joy-generator-google-credentials +stringData: + credentials.json: | + { + "credentials": "from google!" + } +--- +# Source: joy-generator/templates/argocd-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-release-joy-generator-config +data: + token: "$my-release-joy-generator-config:PLUGIN_TOKEN" + baseUrl: "http://my-release-joy-generator" +--- +# Source: joy-generator/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: my-release-joy-generator + labels: + helm.sh/chart: joy-generator-1.0.0 + app.kubernetes.io/name: joy-generator + app.kubernetes.io/instance: my-release + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: joy-generator + app.kubernetes.io/instance: my-release +--- +# Source: joy-generator/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-release-joy-generator + labels: + helm.sh/chart: joy-generator-1.0.0 + app.kubernetes.io/name: joy-generator + app.kubernetes.io/instance: my-release + app.kubernetes.io/version: "1.0.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: joy-generator + app.kubernetes.io/instance: my-release + template: + metadata: + labels: + app.kubernetes.io/name: joy-generator + app.kubernetes.io/instance: my-release + spec: + securityContext: + {} + volumes: + - name: catalog-dir + emptyDir: {} + - name: google-credentials + secret: + secretName: my-release-joy-generator-google-credentials + containers: + - name: joy-generator + securityContext: + {} + image: "ghcr.io/nestoca/joy-generator:0.1.2" + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /tmp/catalog + name: catalog-dir + - mountPath: /etc/joy/secrets + name: google-credentials + env: + - name: CATALOG_DIR + value: "/tmp/catalog" + - name: CATALOG_URL + value: "https://github.com/example/foobar.git" + - name: GH_USER + value: "username" + - name: GRACE_PERIOD + value: "10s" + - name: PORT + value: ":8080" + - name: GOOGLE_CREDENTIALS_FILE + value: /etc/joy/secrets/credentials.json + envFrom: + - secretRef: + name: my-release-joy-generator-config + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /api/v1/health + port: http + readinessProbe: + httpGet: + path: /api/v1/readiness + port: http + resources: + {} diff --git a/chart/tests/with-google-credentials/values.yaml b/chart/tests/with-google-credentials/values.yaml new file mode 100644 index 0000000..a99c74c --- /dev/null +++ b/chart/tests/with-google-credentials/values.yaml @@ -0,0 +1,20 @@ +# Default values for joy-generator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +env: + CATALOG_URL: https://github.com/example/foobar.git + GH_USER: username + +secretEnv: + values: + PLUGIN_TOKEN: "@very!l0ngands3curet0ken" + GH_TOKEN: "12312312312312" + +image: + tag: 0.1.2 + +googleCredentials: | + { + "credentials": "from google!" + } diff --git a/cmd/server/api_get_params_test.go b/cmd/server/api_get_params_test.go index 857ab90..045383e 100644 --- a/cmd/server/api_get_params_test.go +++ b/cmd/server/api_get_params_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -27,10 +28,14 @@ func TestGetParamsE2E(t *testing.T) { } var ( - user github.User - catalog github.RepoMetadata + user github.User + catalog github.RepoMetadata + registry string + credentials []byte ) + conf.Var(conf.Environ, ®istry, "REGISTRY", conf.Required[string](true)) + conf.Var(conf.Environ, &credentials, "CREDENTIALS", conf.Required[[]byte](true)) conf.Var(conf.Environ, &catalog.Path, "CATALOG_PATH", conf.Default(filepath.Join(os.TempDir(), "catalog"))) conf.Var(conf.Environ, &catalog.URL, "CATALOG_URL", conf.Required[string](true)) conf.Var(conf.Environ, &catalog.TargetRevision, "CATALOG_REVISION", conf.Default("master")) @@ -39,6 +44,8 @@ func TestGetParamsE2E(t *testing.T) { require.NoError(t, conf.Environ.Parse()) + require.NoError(t, AuthenticateHelm(context.Background(), registry, credentials)) + require.NoError(t, os.RemoveAll(catalog.Path)) repo, err := user.NewRepo(catalog) @@ -49,13 +56,20 @@ func TestGetParamsE2E(t *testing.T) { repo = repo.WithLogger(logger) + cacheDir, err := os.MkdirTemp("", "joy-cache-*") + require.NoError(t, err) + + t.Logf("cache dir: %s", cacheDir) + handler := Handler(HandlerParams{ pluginToken: "test-token", logger: logger, repo: repo, generator: &generator.Generator{ + CacheRoot: cacheDir, LoadJoyContext: generator.RepoLoader(repo), Logger: logger, + ChartPuller: generator.ChartPuller{}, }, }) @@ -79,7 +93,6 @@ func TestGetParamsE2E(t *testing.T) { var response generator.GetParamsResponse require.NoError(t, json.Unmarshal(body.Bytes(), &response)) - require.Greater(t, len(response.Output.Parameters), 0) for _, result := range response.Output.Parameters { @@ -92,8 +105,12 @@ func TestGetParamsE2E(t *testing.T) { require.Greater(t, len(logs.Records), 0) for _, record := range logs.Records { require.NotEmpty(t, record["level"]) - require.NotEqual(t, "error", record["level"]) + require.NotEqualf(t, "error", record["level"], "unexpected error log: %+v", record) } + + entries, err := os.ReadDir(cacheDir) + require.NoError(t, err) + require.Greater(t, len(entries), 0) } type TestLogOutputs struct { diff --git a/cmd/server/config.go b/cmd/server/config.go index 9da331e..1df8a54 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -18,6 +18,14 @@ type Config struct { Catalog github.RepoMetadata + CacheRoot string + + Google struct { + Repository string + CredentialsFilePath string + RawCredentials []byte + } + Github struct { User github.User App github.App @@ -27,19 +35,37 @@ type Config struct { func GetConfig() Config { var cfg Config + home, err := os.UserHomeDir() + if err != nil { + panic(err) + } + + conf.Var(conf.Environ, &cfg.CacheRoot, "CACHE_ROOT", conf.Default(filepath.Join(home, ".cache", "joy"))) conf.Var(conf.Environ, &cfg.Port, "PORT", conf.Default(":8080")) conf.Var(conf.Environ, &cfg.GracePeriod, "GRACE_PERIOD", conf.Default(10*time.Second)) conf.Var(conf.Environ, &cfg.PluginToken, "PLUGIN_TOKEN") + conf.Var(conf.Environ, &cfg.Catalog.URL, "CATALOG_URL") conf.Var(conf.Environ, &cfg.Catalog.Path, "CATALOG_DIR", conf.Default(filepath.Join(os.TempDir(), "catalog"))) conf.Var(conf.Environ, &cfg.Catalog.TargetRevision, "CATALOG_REVISION") + conf.Var(conf.Environ, &cfg.Github.User.Token, "GH_TOKEN") conf.Var(conf.Environ, &cfg.Github.User.Name, "GH_USER") conf.Var(conf.Environ, &cfg.Github.App.ID, "GH_APP_ID") conf.Var(conf.Environ, &cfg.Github.App.InstallationID, "GH_APP_INSTALLATION_ID") conf.Var(conf.Environ, &cfg.Github.App.PrivateKeyPath, "GH_APP_PRIVATE_KEY_PATH") + conf.Var(conf.Environ, &cfg.Google.CredentialsFilePath, "GOOGLE_CREDENTIALS_FILE") + conf.Var(conf.Environ, &cfg.Google.Repository, "GOOGLE_ARTIFACT_REPOSITORY") + conf.Environ.MustParse() + if path := cfg.Google.CredentialsFilePath; path != "" { + fs := conf.MakeParser(conf.FileSystem(conf.FileSystemOptions{})) + defer fs.MustParse() + + conf.Var(fs, &cfg.Google.RawCredentials, path, conf.Required[[]byte](true)) + } + return cfg } diff --git a/cmd/server/main.go b/cmd/server/main.go index f403aec..441a4e0 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,11 +1,13 @@ package main import ( + "bytes" "context" "errors" "fmt" "net/http" "os" + "os/exec" "reflect" "syscall" "time" @@ -32,6 +34,15 @@ func run() error { cfg := GetConfig() + ctx, stop := xcontext.WithSignalCancelation(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if len(cfg.Google.RawCredentials) > 0 { + if err := AuthenticateHelm(ctx, cfg.Google.Repository, cfg.Google.RawCredentials); err != nil { + return fmt.Errorf("failed to authenticate to helm: %w", err) + } + } + repo, err := func() (*github.Repo, error) { if !reflect.ValueOf(cfg.Github.App).IsZero() { return cfg.Github.App.NewRepo(cfg.Catalog) @@ -53,8 +64,10 @@ func run() error { logger: logger, repo: repo, generator: &generator.Generator{ - Logger: logger, + CacheRoot: cfg.CacheRoot, LoadJoyContext: generator.RepoLoader(repo), + Logger: logger, + ChartPuller: generator.ChartPuller{}, }, }), ReadHeaderTimeout: 5 * time.Second, @@ -69,9 +82,6 @@ func run() error { } }() - ctx, stop := xcontext.WithSignalCancelation(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - select { case err := <-errChan: return fmt.Errorf("failed to listen and serve: %w", err) @@ -89,3 +99,18 @@ func run() error { return nil } + +func AuthenticateHelm(ctx context.Context, registry string, credentials []byte) error { + login := exec.CommandContext(ctx, "helm", "registry", "login", "-u", "_json_key", "--password-stdin", registry) + + var buffer bytes.Buffer + login.Stdout = &buffer + login.Stderr = &buffer + login.Stdin = bytes.NewReader(credentials) + + if err := login.Run(); err != nil { + return fmt.Errorf("%w: %q", err, &buffer) + } + + return nil +} diff --git a/go.mod b/go.mod index 484ca5d..b6b6f93 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,14 @@ require ( github.com/davidmdm/x/xcontext v0.0.2 github.com/gin-gonic/gin v1.9.1 github.com/go-git/go-git/v5 v5.12.0 - github.com/nestoca/joy v0.50.0 + github.com/nestoca/joy v0.50.1-0.20240508173227-9709629cdf30 github.com/rs/zerolog v1.32.0 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + cuelang.org/go v0.7.1 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect @@ -27,6 +28,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/cockroachdb/apd/v3 v3.2.1 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmdm/x/xerr v0.0.3 // indirect @@ -62,6 +64,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect github.com/nestoca/survey/v2 v2.0.0 // indirect github.com/pelletier/go-toml/v2 v2.2.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect diff --git a/go.sum b/go.sum index 6c0289c..525137f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 h1:zkiIe8AxZ/kDjqQN+mDKc5BxoVJOqioSdqApjc+eB1I= +cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13/go.mod h1:XGKYSMtsJWfqQYPwq51ZygxAPqpEUj/9bdg16iDPTAA= +cuelang.org/go v0.7.1 h1:wSuUSIKR9M1yrph57l8EJATWVRWHaq/Zd0dFUL10PC8= +cuelang.org/go v0.7.1/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -39,6 +43,8 @@ github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLI github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= +github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= @@ -58,8 +64,12 @@ github.com/davidmdm/x/xfs v0.0.4 h1:7g0ue0Gfs5mrIaV8QGA03hgIPm3tJvHUVdsr6obQTt4= github.com/davidmdm/x/xfs v0.0.4/go.mod h1:Mpo/6dE+3m6XNFPqai975wPt/XundxCW4cErGpvUSdc= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw= +github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -84,6 +94,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -133,6 +145,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -149,6 +163,8 @@ github.com/mgutz/str v1.2.0/go.mod h1:w1v0ofgLaJdoD0HpQ3fycxKD1WtxpjSo151pK/31q6 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -157,14 +173,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nestoca/joy v0.49.4 h1:KHHy95mTjnkGEUFvJ2euQULVLpaZ3m+x58hNZiFGhEM= -github.com/nestoca/joy v0.49.4/go.mod h1:AYgZ15OI1+9WtkXsa5ubhRO4S/CBiBmRPBDPp6aICLQ= -github.com/nestoca/joy v0.50.0 h1:/AnIbxSaahb7DnBSAHZiT4NOuLeXLlf47BNOXrDcFL4= -github.com/nestoca/joy v0.50.0/go.mod h1:AYgZ15OI1+9WtkXsa5ubhRO4S/CBiBmRPBDPp6aICLQ= +github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto= +github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= +github.com/nestoca/joy v0.50.1-0.20240508173227-9709629cdf30 h1:4H4TGr2X8PMDY7xOeHGQvlNmzLyASUbjLCNYGpRJ9rc= +github.com/nestoca/joy v0.50.1-0.20240508173227-9709629cdf30/go.mod h1:AYgZ15OI1+9WtkXsa5ubhRO4S/CBiBmRPBDPp6aICLQ= github.com/nestoca/survey/v2 v2.0.0 h1:orM/TXtBSQJCPiZy9N51hcwH0WzFjQJ7TQKCfMTnGoQ= github.com/nestoca/survey/v2 v2.0.0/go.mod h1:QfmcQfCRtqsiBpePFhHEW0MY9QH7lSpw3CQinsdC/dc= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= @@ -173,8 +193,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4= +github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= +github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c h1:fPpdjePK1atuOg28PXfNSqgwf9I/qD1Hlo39JFwKBXk= +github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= diff --git a/internal/generator/generator.go b/internal/generator/generator.go index c9494d1..5826fde 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -1,6 +1,8 @@ package generator import ( + "bytes" + "context" "fmt" "github.com/rs/zerolog" @@ -9,13 +11,34 @@ import ( "github.com/nestoca/joy/api/v1alpha1" joy "github.com/nestoca/joy/pkg" "github.com/nestoca/joy/pkg/catalog" + "github.com/nestoca/joy/pkg/helm" "github.com/nestoca/joy-generator/internal/github" ) type Generator struct { + CacheRoot string LoadJoyContext JoyLoaderFunc Logger zerolog.Logger + ChartPuller helm.Puller +} + +type ChartPuller struct{} + +func (puller ChartPuller) Pull(ctx context.Context, opts helm.PullOptions) error { + var buffer bytes.Buffer + + cli := helm.CLI{ + IO: joy.IO{ + Out: &buffer, + Err: &buffer, + }, + } + + if err := cli.Pull(ctx, opts); err != nil { + return fmt.Errorf("%w: %q", err, &buffer) + } + return nil } type Result struct { @@ -67,6 +90,13 @@ func (generator *Generator) Run() ([]Result, error) { return nil, fmt.Errorf("loading joy context: %w", err) } + cache := helm.ChartCache{ + Root: generator.CacheRoot, + Puller: generator.ChartPuller, + Refs: joyctx.Config.Charts, + DefaultChartRef: joyctx.Config.DefaultChartRef, + } + var reconciledReleases []Result for _, crossRelease := range joyctx.Catalog.Releases.Items { for _, release := range crossRelease.Releases { @@ -80,7 +110,7 @@ func (generator *Generator) Run() ([]Result, error) { Str("environment", release.Environment.Name). Msg("processing release") - chart, err := joy.ChartFromRelease(release, joyctx.Config.Charts, joyctx.Config.DefaultChartRef) + chart, err := cache.GetReleaseChartFS(context.TODO(), release) if err != nil { generator.Logger. Error(). @@ -94,7 +124,7 @@ func (generator *Generator) Run() ([]Result, error) { release.Spec.Chart.Name = chart.Name release.Spec.Chart.Version = chart.Version - values, err := joy.ReleaseValues(release, joyctx.Config.ValueMapping) + values, err := joy.ComputeReleaseValues(release, chart, joyctx.Config.ValueMapping) if err != nil { generator.Logger. Error(). diff --git a/internal/generator/generator_test.go b/internal/generator/generator_test.go index e859a66..b0b368c 100644 --- a/internal/generator/generator_test.go +++ b/internal/generator/generator_test.go @@ -8,10 +8,11 @@ import ( "github.com/nestoca/joy/api/v1alpha1" joy "github.com/nestoca/joy/pkg" "github.com/nestoca/joy/pkg/catalog" + "github.com/nestoca/joy/pkg/helm" ) func TestGenerator(t *testing.T) { - baseCharts := map[string]joy.HelmChart{ + baseCharts := map[string]helm.Chart{ "default": { RepoURL: "test", Name: "chart", @@ -28,7 +29,7 @@ func TestGenerator(t *testing.T) { Name string Release *v1alpha1.Release DefaultChartRef string - Charts map[string]joy.HelmChart + Charts map[string]helm.Chart ValueMapping *joy.ValueMapping ExpectedRelease *v1alpha1.Release ExpectedValues string @@ -148,6 +149,7 @@ func TestGenerator(t *testing.T) { for _, tc := range cases { t.Run(tc.Name, func(t *testing.T) { generator := Generator{ + ChartPuller: &helm.PullRendererMock{}, LoadJoyContext: func() (*JoyContext, error) { return &JoyContext{ Catalog: BuildCatalogFromRelease(tc.Release),