Support fetching a piped config from AWS SSM Parameter Store (#5249)

* Generate v0.49.x docs Signed-off-by: t-kikuc <[email protected]> * add --aws-ssm-parameter Signed-off-by: t-kikuc <[email protected]> * add config-aws-ssm-parameter to piped Signed-off-by: t-kikuc <[email protected]> * update docs of runtime options Signed-off-by: t-kikuc <[email protected]> --------- Signed-off-by: t-kikuc <[email protected]>
pipe-cd · Oct 3, 2024 · 2ef9ad3 · 2ef9ad3
1 parent 38d647a
commit 2ef9ad3
Show file tree

Hide file tree

Showing 6 changed files with 156 additions and 71 deletions.
diff --git a/docs/content/en/docs-dev/user-guide/managing-piped/runtime-options.md b/docs/content/en/docs-dev/user-guide/managing-piped/runtime-options.md
@@ -20,6 +20,7 @@ Flags:
       --app-manifest-cache-count int               The number of app manifests to cache. The cache-key contains the commit hash. The default is 150. (default 150)     
       --cert-file string                           The path to the TLS certificate file.
       --config-aws-secret string                   The ARN of secret that contains Piped config and be stored in AWS Secrets Manager.
+      --config-aws-ssm-parameter string            The name of parameter of Piped config stored in AWS Systems Manager Parameter Store. SecureString is also supported.
       --config-data string                         The base64 encoded string of the configuration data.
       --config-file string                         The path to the configuration file.
       --config-gcp-secret string                   The resource ID of secret that contains Piped config and be stored in GCP SecretManager.
@@ -46,25 +47,27 @@ Usage:
   launcher launcher [flags]
 
 Flags:
-      --aws-secret-id string           The ARN of secret that contains Piped config in AWS Secrets Manager service.
-      --cert-file string               The path to the TLS certificate file.
-      --check-interval duration        Interval to periodically check desired config/version to restart Piped. Default is 1m. (default 1m0s)
-      --config-data string             The base64 encoded string of the configuration data.
-      --config-file string             The path to the configuration file.
-      --config-from-aws-secret         Whether to load Piped config that is being stored in AWS Secrets Manager service.
-      --config-from-gcp-secret         Whether to load Piped config that is being stored in GCP SecretManager service.
-      --config-from-git-repo           Whether to load Piped config that is being stored in a git repository.
-      --default-version string         The version should be run when no desired version was specified. Empty means using the same version with Launcher.
-      --gcp-secret-id string           The resource ID of secret that contains Piped config in GCP SecretManager service.
-      --git-branch string              Branch of git repository to for Piped config.
-      --git-piped-config-file string   Relative path within git repository to locate Piped config file.
-      --git-repo-url string            The remote URL of git repository to fetch Piped config.
-      --git-ssh-key-file string        The path to SSH private key to fetch private git repository.
-      --grace-period duration          How long to wait for graceful shutdown. (default 30s)
-  -h, --help                           help for launcher
-      --home-dir string                The working directory of Launcher.
-      --insecure                       Whether disabling transport security while connecting to control-plane.
-      --launcher-admin-port int        The port number used to run a HTTP server for admin tasks such as metrics, healthz.
+      --aws-secret-id string                  The ARN of secret that contains Piped config in AWS Secrets Manager service.
+      --aws-ssm-parameter string              The name of parameter of Piped config stored in AWS Systems Manager Parameter Store. SecureString is also supported.
+      --cert-file string                      The path to the TLS certificate file.
+      --check-interval duration               Interval to periodically check desired config/version to restart Piped. Default is 1m. (default 1m0s)
+      --config-data string                    The base64 encoded string of the configuration data.
+      --config-file string                    The path to the configuration file.
+      --config-from-aws-secret                Whether to load Piped config that is being stored in AWS Secrets Manager service.
+      --config-from-aws-ssm-parameter-store   Whether to load Piped config that is being stored in AWS Systems Manager Parameter Store.
+      --config-from-gcp-secret                Whether to load Piped config that is being stored in GCP SecretManager service.
+      --config-from-git-repo                  Whether to load Piped config that is being stored in a git repository.
+      --default-version string                The version should be run when no desired version was specified. Empty means using the same version with Launcher.
+      --gcp-secret-id string                  The resource ID of secret that contains Piped config in GCP SecretManager service.
+      --git-branch string                     Branch of git repository to for Piped config.
+      --git-piped-config-file string          Relative path within git repository to locate Piped config file.
+      --git-repo-url string                   The remote URL of git repository to fetch Piped config.
+      --git-ssh-key-file string               The path to SSH private key to fetch private git repository.
+      --grace-period duration                 How long to wait for graceful shutdown. (default 30s)
+  -h, --help                                  help for launcher
+      --home-dir string                       The working directory of Launcher.
+      --insecure                              Whether disabling transport security while connecting to control-plane.
+      --launcher-admin-port int               The port number used to run a HTTP server for admin tasks such as metrics, healthz.
 
 Global Flags:
       --log-encoding string                The encoding type for logger [json|console|humanize]. (default "humanize")

diff --git a/go.mod b/go.mod
@@ -18,6 +18,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/lambda v1.62.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.2
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.54.3
 	github.com/coreos/go-oidc/v3 v3.11.0
 	github.com/creasty/defaults v1.6.0
 	github.com/envoyproxy/go-control-plane v0.12.0

diff --git a/go.sum b/go.sum
@@ -149,6 +149,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2 h1:1iXmXy8SJzQVMGvo40TSzBYS9ig6B
 github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.2 h1:C79sbcbdKuBpBpTDy1MNrJx5/Wii7gcwt0Jkd5QCGNA=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.2/go.mod h1:WyLS5qwXHtjKAONYZq/4ewdd+hcVsa3LBu77Ow5uj3k=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.54.3 h1:Ctzev3ppcc46m2FgrLEZhsHMEr1G1lrJcd9Cmoy/QJk=
+github.com/aws/aws-sdk-go-v2/service/ssm v1.54.3/go.mod h1:qs3TBNpFEnVubl0WL3jruj7NJMF1RCAPEPQ1f+fLTBE=
 github.com/aws/aws-sdk-go-v2/service/sso v1.23.2 h1:yzi/y/vKlLyzOfG7pSu5ONNGRxHIgLeDrV4w2AMRCo0=
 github.com/aws/aws-sdk-go-v2/service/sso v1.23.2/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2 h1:3gb6pYhYLjo8rB1h2Tqs61wpjRd3rQymYcVq/pp0yxI=

diff --git a/pkg/app/launcher/cmd/launcher/launcher.go b/pkg/app/launcher/cmd/launcher/launcher.go
@@ -30,8 +30,10 @@ import (
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	secretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	awssecretsmanager "github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	awsssm "github.com/aws/aws-sdk-go-v2/service/ssm"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
@@ -58,24 +60,26 @@ const (
 var ignoreFlags map[string]struct{}
 
 type launcher struct {
-	configFile          string
-	configData          string
-	configFromGCPSecret bool
-	gcpSecretID         string
-	configFromAWSSecret bool
-	awsSecretID         string
-	configFromGitRepo   bool
-	gitRepoURL          string
-	gitBranch           string
-	gitPipedConfigFile  string
-	gitSSHKeyFile       string
-	insecure            bool
-	certFile            string
-	homeDir             string
-	defaultVersion      string
-	launcherAdminPort   int
-	checkInterval       time.Duration
-	gracePeriod         time.Duration
+	configFile                     string
+	configData                     string
+	configFromGCPSecret            bool
+	gcpSecretID                    string
+	configFromAWSSecret            bool
+	awsSecretID                    string
+	configFromAWSSsmParameterStore bool
+	awsSsmParameter                string
+	configFromGitRepo              bool
+	gitRepoURL                     string
+	gitBranch                      string
+	gitPipedConfigFile             string
+	gitSSHKeyFile                  string
+	insecure                       bool
+	certFile                       string
+	homeDir                        string
+	defaultVersion                 string
+	launcherAdminPort              int
+	checkInterval                  time.Duration
+	gracePeriod                    time.Duration
 
 	runningVersion    string
 	runningConfigData []byte
@@ -108,6 +112,9 @@ func NewCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&l.configFromAWSSecret, "config-from-aws-secret", l.configFromAWSSecret, "Whether to load Piped config that is being stored in AWS Secrets Manager service.")
 	cmd.Flags().StringVar(&l.awsSecretID, "aws-secret-id", l.awsSecretID, "The ARN of secret that contains Piped config in AWS Secrets Manager service.")
 
+	cmd.Flags().BoolVar(&l.configFromAWSSsmParameterStore, "config-from-aws-ssm-parameter-store", l.configFromAWSSsmParameterStore, "Whether to load Piped config that is being stored in AWS Systems Manager Parameter Store.")
+	cmd.Flags().StringVar(&l.awsSsmParameter, "aws-ssm-parameter", l.awsSsmParameter, "The name of parameter of Piped config stored in AWS Systems Manager Parameter Store. SecureString is also supported.")
+
 	cmd.Flags().BoolVar(&l.configFromGitRepo, "config-from-git-repo", l.configFromGitRepo, "Whether to load Piped config that is being stored in a git repository.")
 	cmd.Flags().StringVar(&l.gitRepoURL, "git-repo-url", l.gitRepoURL, "The remote URL of git repository to fetch Piped config.")
 	cmd.Flags().StringVar(&l.gitBranch, "git-branch", l.gitBranch, "Branch of git repository to for Piped config.")
@@ -126,21 +133,23 @@ func NewCommand() *cobra.Command {
 
 	// TODO: Find a better way to automatically maintain this ignore list.
 	ignoreFlags = map[string]struct{}{
-		"config-file":            {},
-		"config-data":            {},
-		"config-from-gcp-secret": {},
-		"gcp-secret-id":          {},
-		"config-from-git-repo":   {},
-		"config-from-aws-secret": {},
-		"aws-secret-id":          {},
-		"git-repo-url":           {},
-		"git-branch":             {},
-		"git-piped-config-file":  {},
-		"git-ssh-key-file":       {},
-		"home-dir":               {},
-		"default-version":        {},
-		"launcher-admin-port":    {},
-		"check-interval":         {},
+		"config-file":                         {},
+		"config-data":                         {},
+		"config-from-gcp-secret":              {},
+		"gcp-secret-id":                       {},
+		"config-from-git-repo":                {},
+		"config-from-aws-secret":              {},
+		"aws-secret-id":                       {},
+		"config-from-aws-ssm-parameter-store": {},
+		"aws-ssm-parameter":                   {},
+		"git-repo-url":                        {},
+		"git-branch":                          {},
+		"git-piped-config-file":               {},
+		"git-ssh-key-file":                    {},
+		"home-dir":                            {},
+		"default-version":                     {},
+		"launcher-admin-port":                 {},
+		"check-interval":                      {},
 	}
 
 	return cmd
@@ -157,6 +166,11 @@ func (l *launcher) validateFlags() error {
 			return fmt.Errorf("aws-secret-id must be set to load Piped config from AWS Secrets Manager service")
 		}
 	}
+	if l.configFromAWSSsmParameterStore {
+		if l.awsSsmParameter == "" {
+			return fmt.Errorf("aws-ssm-parameter must be set to load Piped config from AWS Systems Manager Parameter Store")
+		}
+	}
 	if l.configFromGitRepo {
 		if l.gitRepoURL == "" {
 			return fmt.Errorf("git-repo-url must be set to load config from a git repository")
@@ -455,6 +469,27 @@ func (l *launcher) loadConfigData(ctx context.Context) ([]byte, error) {
 		return decoded, nil
 	}
 
+	if l.configFromAWSSsmParameterStore {
+		cfg, err := awsconfig.LoadDefaultConfig(ctx)
+		if err != nil {
+			return nil, err
+		}
+		client := awsssm.NewFromConfig(cfg)
+		in := &awsssm.GetParameterInput{
+			Name:           &l.awsSsmParameter,
+			WithDecryption: aws.Bool(true),
+		}
+		result, err := client.GetParameter(ctx, in)
+		if err != nil {
+			return nil, err
+		}
+		decoded, err := base64.StdEncoding.DecodeString(*result.Parameter.Value)
+		if err != nil {
+			return nil, err
+		}
+		return decoded, nil
+	}
+
 	if l.configFromGitRepo {
 		// Pull to update the local data.
 		if err := l.configRepo.Pull(ctx, l.gitBranch); err != nil {
@@ -468,6 +503,7 @@ func (l *launcher) loadConfigData(ctx context.Context) ([]byte, error) {
 		"config-data",
 		"config-from-gcp-secret",
 		"config-from-aws-secret",
+		"config-from-aws-ssm-parameter-store",
 		"config-from-git-repo",
 	}, ", "))
 }

diff --git a/pkg/app/piped/cmd/piped/piped.go b/pkg/app/piped/cmd/piped/piped.go
@@ -31,8 +31,10 @@ import (
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	secretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+	"github.com/aws/aws-sdk-go-v2/aws"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	awssecretsmanager "github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	awsssm "github.com/aws/aws-sdk-go-v2/service/ssm"
 	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
@@ -92,10 +94,11 @@ const (
 )
 
 type piped struct {
-	configFile      string
-	configData      string
-	configGCPSecret string
-	configAWSSecret string
+	configFile            string
+	configData            string
+	configGCPSecret       string
+	configAWSSecret       string
+	configAWSSsmParameter string
 
 	insecure                             bool
 	certFile                             string
@@ -131,6 +134,7 @@ func NewCommand() *cobra.Command {
 	cmd.Flags().StringVar(&p.configData, "config-data", p.configData, "The base64 encoded string of the configuration data.")
 	cmd.Flags().StringVar(&p.configGCPSecret, "config-gcp-secret", p.configGCPSecret, "The resource ID of secret that contains Piped config and be stored in GCP SecretManager.")
 	cmd.Flags().StringVar(&p.configAWSSecret, "config-aws-secret", p.configAWSSecret, "The ARN of secret that contains Piped config and be stored in AWS Secrets Manager.")
+	cmd.Flags().StringVar(&p.configAWSSsmParameter, "config-aws-ssm-parameter", p.configAWSSsmParameter, "The name of parameter of Piped config stored in AWS Systems Manager Parameter Store. SecureString is also supported.")
 
 	cmd.Flags().BoolVar(&p.insecure, "insecure", p.insecure, "Whether disabling transport security while connecting to control-plane.")
 	cmd.Flags().StringVar(&p.certFile, "cert-file", p.certFile, "The path to the TLS certificate file.")
@@ -735,6 +739,18 @@ func (p *piped) loadConfig(ctx context.Context) (*config.PipedSpec, error) {
 		return extract(cfg)
 	}
 
+	if p.configAWSSsmParameter != "" {
+		data, err := p.getConfigDataFromAWSSsmParameterStore(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load config from AWS Systems Manager Parameter Store (%w)", err)
+		}
+		cfg, err := config.DecodeYAML(data)
+		if err != nil {
+			return nil, err
+		}
+		return extract(cfg)
+	}
+
 	return nil, fmt.Errorf("one of config-file, config-gcp-secret or config-aws-secret must be set")
 }
 
@@ -935,6 +951,29 @@ func (p *piped) getConfigDataFromAWSSecretsManager(ctx context.Context) ([]byte,
 	return decoded, nil
 }
 
+func (p *piped) getConfigDataFromAWSSsmParameterStore(ctx context.Context) ([]byte, error) {
+	cfg, err := awsconfig.LoadDefaultConfig(ctx)
+	if err != nil {
+		return nil, err
+	}
+	client := awsssm.NewFromConfig(cfg)
+
+	in := &awsssm.GetParameterInput{
+		Name:           &p.configAWSSsmParameter,
+		WithDecryption: aws.Bool(true),
+	}
+	result, err := client.GetParameter(ctx, in)
+	if err != nil {
+		return nil, err
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(*result.Parameter.Value)
+	if err != nil {
+		return nil, err
+	}
+	return decoded, nil
+}
+
 func registerMetrics(pipedID, projectID, launcherVersion string) *prometheus.Registry {
 	r := prometheus.NewRegistry()
 	wrapped := prometheus.WrapRegistererWith(
@@ -1009,13 +1048,13 @@ func stopCommandHandler(ctx context.Context, cmdLister commandstore.Lister, logg
 
 func (p *piped) hasTooManyConfigFlags() error {
 	cnt := 0
-	for _, v := range []string{p.configFile, p.configGCPSecret, p.configAWSSecret} {
+	for _, v := range []string{p.configFile, p.configGCPSecret, p.configAWSSecret, p.configAWSSsmParameter} {
 		if v != "" {
 			cnt++
 		}
 	}
 	if cnt > 1 {
-		return fmt.Errorf("only one of config-file, config-gcp-secret or config-aws-secret could be set")
+		return fmt.Errorf("only one of config-file, config-gcp-secret, config-aws-secret, or config-aws-ssm-parameter could be set")
 	}
 	return nil
 }
diff --git a/pkg/app/piped/cmd/piped/piped_test.go b/pkg/app/piped/cmd/piped/piped_test.go
@@ -31,36 +31,40 @@ func TestHasTooManyConfigFlags(t *testing.T) {
 		{
 			title: "no config",
 			p: &piped{
-				configFile:      "",
-				configGCPSecret: "",
-				configAWSSecret: "",
+				configFile:            "",
+				configGCPSecret:       "",
+				configAWSSecret:       "",
+				configAWSSsmParameter: "",
 			},
 			expectErr: false,
 		},
 		{
 			title: "only one config is set",
 			p: &piped{
-				configFile:      "config.yaml",
-				configGCPSecret: "",
-				configAWSSecret: "",
+				configFile:            "config.yaml",
+				configGCPSecret:       "",
+				configAWSSecret:       "",
+				configAWSSsmParameter: "",
 			},
 			expectErr: false,
 		},
 		{
 			title: "two configs are set",
 			p: &piped{
-				configFile:      "config.yaml",
-				configGCPSecret: "xxx",
-				configAWSSecret: "",
+				configFile:            "config.yaml",
+				configGCPSecret:       "xxx",
+				configAWSSecret:       "",
+				configAWSSsmParameter: "",
 			},
 			expectErr: true,
 		},
 		{
-			title: "three configs are set",
+			title: "all configs are set",
 			p: &piped{
-				configFile:      "config.yaml",
-				configGCPSecret: "xxx",
-				configAWSSecret: "yyy",
+				configFile:            "config.yaml",
+				configGCPSecret:       "xxx",
+				configAWSSecret:       "yyy",
+				configAWSSsmParameter: "zzz",
 			},
 			expectErr: true,
 		},