From 33d139828af33f32e72de8eafcb2fb3610490ab6 Mon Sep 17 00:00:00 2001 From: Piyush Garg Date: Mon, 16 Sep 2024 01:25:14 +0530 Subject: [PATCH] Add support for multiple git resolver configurations This will add support for providing multiple configurations in git resolver configmap Configurations can be provided using unique key and that key can be passed as a param in the taskrun/pipelinerun to resolver Old format is supported to provide backward compatibility and docs added in details about how to use the feature. Unit and e2e test added Fixes #5487 --- docs/git-resolver.md | 115 +++++++- .../resolver/framework/fakeresolver.go | 4 +- .../resolver/framework/reconciler.go | 11 +- pkg/remoteresolution/resolver/git/resolver.go | 16 +- .../resolver/git/resolver_test.go | 88 +++++- .../resolver/framework/fakeresolver.go | 4 +- .../resolver/framework/interface.go | 2 +- .../resolver/framework/reconciler.go | 11 +- pkg/resolution/resolver/git/config.go | 61 ++++ pkg/resolution/resolver/git/config_test.go | 147 ++++++++++ pkg/resolution/resolver/git/params.go | 2 + pkg/resolution/resolver/git/resolver.go | 94 ++++-- pkg/resolution/resolver/git/resolver_test.go | 272 +++++++++++++++++- test/resolvers_test.go | 64 +++++ 14 files changed, 828 insertions(+), 63 deletions(-) create mode 100644 pkg/resolution/resolver/git/config_test.go diff --git a/docs/git-resolver.md b/docs/git-resolver.md index 7ec56fda7da..6593377e604 100644 --- a/docs/git-resolver.md +++ b/docs/git-resolver.md @@ -114,10 +114,6 @@ Note that not all `go-scm` implementations have been tested with the `git` resol * BitBucket Server * BitBucket Cloud -Fetching from multiple Git providers with different configuration is not -supported. You can use the [http resolver](./http-resolver.md) to fetch URL -from another provider with different credentials. - #### Task Resolution ```yaml @@ -195,6 +191,117 @@ spec: value: Ranni ``` +### Specifying Configuration for Multiple Git Providers + +It is possible to specify configurations for multiple providers and even multiple configurations for same provider to use in +different tekton resources. You need to first add details in configmap with your unique identifier key prefix. +To use them in tekton resources, pass the unique key mentioned in configmap as an extra param to resolver with key +`configKey` and value your unique key. If no `configKey` param is passed, `default` will be used. You can specify +default configuration in configmap either by mentioning no unique identifier or by using default identifier `default` + +**Note**: `configKey` should not contain `.` while specifying configurations in configmap + +### Example Configmap + +You can add multiple configuration to `git-resolver-configmap` like this. All keys mentioned above are supported. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: git-resolver-config + namespace: tekton-pipelines-resolvers + labels: + app.kubernetes.io/component: resolvers + app.kubernetes.io/instance: default + app.kubernetes.io/part-of: tekton-pipelines +data: + # configuration 1, default one to use if no configKey provided or provided with value default + fetch-timeout: "1m" + default-url: "https://github.com/tektoncd/catalog.git" + default-revision: "main" + scm-type: "github" + server-url: "" + api-token-secret-name: "" + api-token-secret-key: "" + api-token-secret-namespace: "default" + default-org: "" + + # configuration 2, will be used if configKey param passed with value test1 + test1.fetch-timeout: "5m" + test1.default-url: "" + test1.default-revision: "stable" + test1.scm-type: "github" + test1.server-url: "api.internal-github.com" + test1.api-token-secret-name: "test1-secret" + test1.api-token-secret-key: "token" + test1.api-token-secret-namespace: "test1" + test1.default-org: "tektoncd" + + # configuration 3, will be used if configKey param passed with value test2 + test2.fetch-timeout: "10m" + test2.default-url: "" + test2.default-revision: "stable" + test2.scm-type: "gitlab" + test2.server-url: "api.internal-gitlab.com" + test2.api-token-secret-name: "test2-secret" + test2.api-token-secret-key: "pat" + test2.api-token-secret-namespace: "test2" + test2.default-org: "tektoncd-infra" +``` + +#### Task Resolution + +A specific configurations from the configMap can be selected by passing the parameter `configKey` with the value +matching one of the configuration keys used in the configMaps. + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: git-api-demo-tr +spec: + taskRef: + resolver: git + params: + - name: org + value: tektoncd + - name: repo + value: catalog + - name: revision + value: main + - name: pathInRepo + value: task/git-clone/0.6/git-clone.yaml + - name: configKey + value: test1 +``` + +#### Pipeline resolution + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: git-api-demo-pr +spec: + pipelineRef: + resolver: git + params: + - name: org + value: tektoncd + - name: repo + value: catalog + - name: revision + value: main + - name: pathInRepo + value: pipeline/simple/0.1/simple.yaml + - name: configKey + value: test2 + params: + - name: name + value: Ranni +``` + ## `ResolutionRequest` Status `ResolutionRequest.Status.RefSource` field captures the source where the remote resource came from. It includes the 3 subfields: `url`, `digest` and `entrypoint`. - `url` diff --git a/pkg/remoteresolution/resolver/framework/fakeresolver.go b/pkg/remoteresolution/resolver/framework/fakeresolver.go index ad7da0e6b85..046ec12f740 100644 --- a/pkg/remoteresolution/resolver/framework/fakeresolver.go +++ b/pkg/remoteresolution/resolver/framework/fakeresolver.go @@ -84,6 +84,6 @@ func (r *FakeResolver) Resolve(_ context.Context, req *v1beta1.ResolutionRequest var _ framework.TimedResolution = &FakeResolver{} // GetResolutionTimeout returns the configured timeout for the reconciler, or the default time.Duration if not configured. -func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - return framework.GetResolutionTimeout(r.Timeout, defaultTimeout) +func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + return framework.GetResolutionTimeout(r.Timeout, defaultTimeout), nil } diff --git a/pkg/remoteresolution/resolver/framework/reconciler.go b/pkg/remoteresolution/resolver/framework/reconciler.go index 96a3d223132..4e35557fe47 100644 --- a/pkg/remoteresolution/resolver/framework/reconciler.go +++ b/pkg/remoteresolution/resolver/framework/reconciler.go @@ -118,9 +118,18 @@ func (r *Reconciler) resolve(ctx context.Context, key string, rr *v1beta1.Resolu errChan := make(chan error) resourceChan := make(chan framework.ResolvedResource) + paramsMap := make(map[string]string) + for _, p := range rr.Spec.Params { + paramsMap[p.Name] = p.Value.StringVal + } + timeoutDuration := defaultMaximumResolutionDuration if timed, ok := r.resolver.(framework.TimedResolution); ok { - timeoutDuration = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration) + var err error + timeoutDuration, err = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration, paramsMap) + if err != nil { + return err + } } // A new context is created for resolution so that timeouts can diff --git a/pkg/remoteresolution/resolver/git/resolver.go b/pkg/remoteresolution/resolver/git/resolver.go index 7231f4b007c..3a3734d4fbf 100644 --- a/pkg/remoteresolution/resolver/git/resolver.go +++ b/pkg/remoteresolution/resolver/git/resolver.go @@ -141,13 +141,17 @@ var _ resolutionframework.TimedResolution = &Resolver{} // GetResolutionTimeout returns a time.Duration for the amount of time a // single git fetch may take. This can be configured with the // fetch-timeout field in the git-resolver-config configmap. -func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - conf := resolutionframework.GetResolverConfigFromContext(ctx) - if timeoutString, ok := conf[git.DefaultTimeoutKey]; ok { +func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + conf, err := git.GetScmConfigForParamConfigKey(ctx, params) + if err != nil { + return time.Duration(0), err + } + if timeoutString := conf.Timeout; timeoutString != "" { timeout, err := time.ParseDuration(timeoutString) - if err == nil { - return timeout + if err != nil { + return time.Duration(0), err } + return timeout, nil } - return defaultTimeout + return defaultTimeout, nil } diff --git a/pkg/remoteresolution/resolver/git/resolver_test.go b/pkg/remoteresolution/resolver/git/resolver_test.go index 18c3e629e70..c1b043983c7 100644 --- a/pkg/remoteresolution/resolver/git/resolver_test.go +++ b/pkg/remoteresolution/resolver/git/resolver_test.go @@ -219,7 +219,10 @@ func TestValidateParams_Failure(t *testing.T) { func TestGetResolutionTimeoutDefault(t *testing.T) { resolver := Resolver{} defaultTimeout := 30 * time.Minute - timeout := resolver.GetResolutionTimeout(context.Background(), defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(context.Background(), defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != defaultTimeout { t.Fatalf("expected default timeout to be returned") } @@ -233,12 +236,34 @@ func TestGetResolutionTimeoutCustom(t *testing.T) { gitresolution.DefaultTimeoutKey: configTimeout.String(), } ctx := resolutionframework.InjectResolverConfigToContext(context.Background(), config) - timeout := resolver.GetResolutionTimeout(ctx, defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != configTimeout { t.Fatalf("expected timeout from config to be returned") } } +func TestGetResolutionTimeoutCustomIdentifier(t *testing.T) { + resolver := Resolver{} + defaultTimeout := 30 * time.Minute + configTimeout := 5 * time.Second + identifierConfigTImeout := 10 * time.Second + config := map[string]string{ + gitresolution.DefaultTimeoutKey: configTimeout.String(), + "foo." + gitresolution.DefaultTimeoutKey: identifierConfigTImeout.String(), + } + ctx := resolutionframework.InjectResolverConfigToContext(context.Background(), config) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{"configKey": "foo"}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } + if timeout != identifierConfigTImeout { + t.Fatalf("expected timeout from config to be returned") + } +} + func TestResolveNotEnabled(t *testing.T) { resolver := Resolver{} @@ -268,6 +293,7 @@ type params struct { namespace string serverURL string scmType string + configKey string } func TestResolve(t *testing.T) { @@ -344,6 +370,7 @@ func TestResolve(t *testing.T) { expectedCommitSHA string expectedStatus *v1beta1.ResolutionRequestStatus expectedErr error + configIdentifer string }{{ name: "clone: default revision main", args: ¶ms{ @@ -439,6 +466,46 @@ func TestResolve(t *testing.T) { apiToken: "some-token", expectedCommitSHA: commitSHAsInSCMRepo[0], expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task from params api information with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + token: "token-secret", + tokenKey: "token", + namespace: "foo", + configKey: "test", + }, + config: map[string]string{ + "test." + gitresolution.ServerURLKey: "fake", + "test." + gitresolution.SCMTypeKey: "fake", + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + configKey: "test", + }, + config: map[string]string{ + "test." + gitresolution.ServerURLKey: "fake", + "test." + gitresolution.SCMTypeKey: "fake", + "test." + gitresolution.APISecretNameKey: "token-secret", + "test." + gitresolution.APISecretKeyKey: "token", + "test." + gitresolution.APISecretNamespaceKey: system.Namespace(), + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), }, { name: "api: successful pipeline", args: ¶ms{ @@ -591,9 +658,9 @@ func TestResolve(t *testing.T) { if cfg == nil { cfg = make(map[string]string) } - cfg[gitresolution.DefaultTimeoutKey] = "1m" - if cfg[gitresolution.DefaultRevisionKey] == "" { - cfg[gitresolution.DefaultRevisionKey] = plumbing.Master.Short() + cfg[tc.configIdentifer+gitresolution.DefaultTimeoutKey] = "1m" + if cfg[tc.configIdentifer+gitresolution.DefaultRevisionKey] == "" { + cfg[tc.configIdentifer+gitresolution.DefaultRevisionKey] = plumbing.Master.Short() } request := createRequest(tc.args) @@ -654,8 +721,8 @@ func TestResolve(t *testing.T) { frtesting.RunResolverReconcileTest(ctx, t, d, resolver, request, expectedStatus, tc.expectedErr, func(resolver framework.Resolver, testAssets test.Assets) { var secretName, secretNameKey, secretNamespace string - if tc.config[gitresolution.APISecretNameKey] != "" && tc.config[gitresolution.APISecretNamespaceKey] != "" && tc.config[gitresolution.APISecretKeyKey] != "" && tc.apiToken != "" { - secretName, secretNameKey, secretNamespace = tc.config[gitresolution.APISecretNameKey], tc.config[gitresolution.APISecretKeyKey], tc.config[gitresolution.APISecretNamespaceKey] + if tc.config[tc.configIdentifer+gitresolution.APISecretNameKey] != "" && tc.config[tc.configIdentifer+gitresolution.APISecretNamespaceKey] != "" && tc.config[tc.configIdentifer+gitresolution.APISecretKeyKey] != "" && tc.apiToken != "" { + secretName, secretNameKey, secretNamespace = tc.config[tc.configIdentifer+gitresolution.APISecretNameKey], tc.config[tc.configIdentifer+gitresolution.APISecretKeyKey], tc.config[tc.configIdentifer+gitresolution.APISecretNamespaceKey] } if tc.args.token != "" && tc.args.namespace != "" && tc.args.tokenKey != "" { secretName, secretNameKey, secretNamespace = tc.args.token, tc.args.tokenKey, tc.args.namespace @@ -879,6 +946,13 @@ func createRequest(args *params) *v1beta1.ResolutionRequest { } } + if args.configKey != "" { + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: gitresolution.ConfigKeyParam, + Value: *pipelinev1.NewStructuredValues(args.configKey), + }) + } + return rr } diff --git a/pkg/resolution/resolver/framework/fakeresolver.go b/pkg/resolution/resolver/framework/fakeresolver.go index 3fd363f825c..b22349d6f56 100644 --- a/pkg/resolution/resolver/framework/fakeresolver.go +++ b/pkg/resolution/resolver/framework/fakeresolver.go @@ -166,8 +166,8 @@ func Resolve(params []pipelinev1.Param, forParam map[string]*FakeResolvedResourc var _ TimedResolution = &FakeResolver{} // GetResolutionTimeout returns the configured timeout for the reconciler, or the default time.Duration if not configured. -func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - return GetResolutionTimeout(r.Timeout, defaultTimeout) +func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + return GetResolutionTimeout(r.Timeout, defaultTimeout), nil } // GetResolutionTimeout returns the input timeout if set to something greater than 0 or the default time.Duration if not configured. diff --git a/pkg/resolution/resolver/framework/interface.go b/pkg/resolution/resolver/framework/interface.go index 39e1fbfba06..89c1e897a77 100644 --- a/pkg/resolution/resolver/framework/interface.go +++ b/pkg/resolution/resolver/framework/interface.go @@ -90,7 +90,7 @@ type TimedResolution interface { // object, which includes any request-scoped data like // resolver config and the request's originating namespace, // along with a default. - GetResolutionTimeout(ctx context.Context, timeout time.Duration) time.Duration + GetResolutionTimeout(ctx context.Context, timeout time.Duration, params map[string]string) (time.Duration, error) } // ResolvedResource returns the data and annotations of a successful diff --git a/pkg/resolution/resolver/framework/reconciler.go b/pkg/resolution/resolver/framework/reconciler.go index 4950a90c74d..314d801c2b1 100644 --- a/pkg/resolution/resolver/framework/reconciler.go +++ b/pkg/resolution/resolver/framework/reconciler.go @@ -109,9 +109,18 @@ func (r *Reconciler) resolve(ctx context.Context, key string, rr *v1beta1.Resolu errChan := make(chan error) resourceChan := make(chan ResolvedResource) + paramsMap := make(map[string]string) + for _, p := range rr.Spec.Params { + paramsMap[p.Name] = p.Value.StringVal + } + timeoutDuration := defaultMaximumResolutionDuration if timed, ok := r.resolver.(TimedResolution); ok { - timeoutDuration = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration) + var err error + timeoutDuration, err = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration, paramsMap) + if err != nil { + return err + } } // A new context is created for resolution so that timeouts can diff --git a/pkg/resolution/resolver/git/config.go b/pkg/resolution/resolver/git/config.go index 44645b1fae6..975166d637a 100644 --- a/pkg/resolution/resolver/git/config.go +++ b/pkg/resolution/resolver/git/config.go @@ -16,6 +16,15 @@ limitations under the License. package git +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" +) + const ( // DefaultTimeoutKey is the configuration field name for controlling // the maximum duration of a resolution request for a file from git. @@ -43,3 +52,55 @@ const ( // APISecretNamespaceKey is the config map key for the token secret's namespace APISecretNamespaceKey = "api-token-secret-namespace" ) + +type GitResolverConfig map[string]ScmConfig + +type ScmConfig struct { + Timeout string `json:"fetch-timeout"` + URL string `json:"default-url"` + Revision string `json:"default-revision"` + Org string `json:"default-org"` + ServerURL string `json:"server-url"` + SCMType string `json:"scm-type"` + APISecretName string `json:"api-token-secret-name"` + APISecretKey string `json:"api-token-secret-key"` + APISecretNamespace string `json:"api-token-secret-namespace"` +} + +func GetGitResolverConfig(ctx context.Context) (GitResolverConfig, error) { + var scmConfig interface{} = &ScmConfig{} + structType := reflect.TypeOf(scmConfig).Elem() + gitResolverConfig := map[string]ScmConfig{} + conf := framework.GetResolverConfigFromContext(ctx) + for key, value := range conf { + var configIdentifier, configKey string + splittedKeyName := strings.Split(key, ".") + switch len(splittedKeyName) { + case 2: + configKey = splittedKeyName[1] + configIdentifier = splittedKeyName[0] + case 1: + configKey = key + configIdentifier = "default" + default: + return nil, fmt.Errorf("key %s passed in git resolver configmap is invalid", key) + } + _, ok := gitResolverConfig[configIdentifier] + if !ok { + gitResolverConfig[configIdentifier] = ScmConfig{} + } + for i := range structType.NumField() { + field := structType.Field(i) + fieldName := field.Name + jsonTag := field.Tag.Get("json") + if configKey == jsonTag { + tokenDetails := gitResolverConfig[configIdentifier] + var scm interface{} = &tokenDetails + structValue := reflect.ValueOf(scm).Elem() + structValue.FieldByName(fieldName).SetString(value) + gitResolverConfig[configIdentifier] = structValue.Interface().(ScmConfig) + } + } + } + return gitResolverConfig, nil +} diff --git a/pkg/resolution/resolver/git/config_test.go b/pkg/resolution/resolver/git/config_test.go new file mode 100644 index 00000000000..b3ec9504afd --- /dev/null +++ b/pkg/resolution/resolver/git/config_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2024 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + resolutionframework "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" + "github.com/tektoncd/pipeline/test/diff" +) + +func TestGetGitResolverConfig(t *testing.T) { + tests := []struct { + name string + wantErr bool + expectedErr string + config map[string]string + expectedConfig GitResolverConfig + }{ + { + name: "no config", + config: map[string]string{}, + expectedConfig: GitResolverConfig{}, + }, + { + name: "default config", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + DefaultOrgKey: "tektoncd", + }, + expectedConfig: GitResolverConfig{ + "default": ScmConfig{ + URL: "https://github.com", + Revision: "main", + Org: "tektoncd", + }, + }, + }, + { + name: "default config with default key", + config: map[string]string{ + "default." + DefaultURLKey: "https://github.com", + "default." + DefaultRevisionKey: "main", + }, + expectedConfig: GitResolverConfig{ + "default": ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + }, + }, + { + name: "config with custom key", + config: map[string]string{ + "test." + DefaultURLKey: "https://github.com", + "test." + DefaultRevisionKey: "main", + }, + expectedConfig: GitResolverConfig{ + "test": ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + }, + }, + { + name: "config with custom key and no key", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github.com", + "test." + DefaultRevisionKey: "main", + }, + expectedConfig: GitResolverConfig{ + "default": ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + "test": ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + }, + }, + { + name: "config with both default and custom key", + config: map[string]string{ + "default." + DefaultURLKey: "https://github.com", + "default." + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github.com", + "test." + DefaultRevisionKey: "main", + }, + expectedConfig: GitResolverConfig{ + "default": ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + "test": ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + }, + }, + { + name: "config with invalid format", + config: map[string]string{ + "default.." + DefaultURLKey: "https://github.com", + }, + wantErr: true, + expectedErr: "key default..default-url passed in git resolver configmap is invalid", + expectedConfig: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := resolutionframework.InjectResolverConfigToContext(context.Background(), tc.config) + gitResolverConfig, err := GetGitResolverConfig(ctx) + if tc.wantErr { + if err == nil { + t.Fatalf("unexpected error parsing git resolver config: %v", err) + } + if d := cmp.Diff(tc.expectedErr, err.Error()); d != "" { + t.Errorf("unexpected error: %s", diff.PrintWantGot(d)) + } + } + if d := cmp.Diff(tc.expectedConfig, gitResolverConfig); d != "" { + t.Errorf("expected config: %s", diff.PrintWantGot(d)) + } + }) + } +} diff --git a/pkg/resolution/resolver/git/params.go b/pkg/resolution/resolver/git/params.go index f1885f7c2b0..d7cd114d8d6 100644 --- a/pkg/resolution/resolver/git/params.go +++ b/pkg/resolution/resolver/git/params.go @@ -39,4 +39,6 @@ const ( ScmTypeParam string = "scmType" // serverURLParam is an optional string to the server URL for the SCM API to connect to ServerURLParam string = "serverURL" + // ConfigKeyParam is an optional string to provid which scm configuration to use from git resolver configmap + ConfigKeyParam string = "configKey" ) diff --git a/pkg/resolution/resolver/git/resolver.go b/pkg/resolution/resolver/git/resolver.go index b0226e2bebc..b20104acc7e 100644 --- a/pkg/resolution/resolver/git/resolver.go +++ b/pkg/resolution/resolver/git/resolver.go @@ -158,20 +158,21 @@ func validateRepoURL(url string) bool { } func ResolveAnonymousGit(ctx context.Context, params map[string]string) (framework.ResolvedResource, error) { - conf := framework.GetResolverConfigFromContext(ctx) + conf, err := GetScmConfigForParamConfigKey(ctx, params) + if err != nil { + return nil, err + } repo := params[UrlParam] if repo == "" { - if urlString, ok := conf[DefaultURLKey]; ok { - repo = urlString - } else { + urlString := conf.URL + if urlString == "" { return nil, errors.New("default Git Repo Url was not set during installation of the git resolver") } } revision := params[RevisionParam] if revision == "" { - if revisionString, ok := conf[DefaultRevisionKey]; ok { - revision = revisionString - } else { + revisionString := conf.Revision + if revisionString == "" { return nil, errors.New("default Git Revision was not set during installation of the git resolver") } } @@ -247,29 +248,37 @@ var _ framework.TimedResolution = &Resolver{} // GetResolutionTimeout returns a time.Duration for the amount of time a // single git fetch may take. This can be configured with the // fetch-timeout field in the git-resolver-config configmap. -func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - conf := framework.GetResolverConfigFromContext(ctx) - if timeoutString, ok := conf[DefaultTimeoutKey]; ok { +func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + conf, err := GetScmConfigForParamConfigKey(ctx, params) + if err != nil { + return time.Duration(0), err + } + if timeoutString := conf.Timeout; timeoutString != "" { timeout, err := time.ParseDuration(timeoutString) - if err == nil { - return timeout + if err != nil { + return time.Duration(0), err } + return timeout, nil } - return defaultTimeout + return defaultTimeout, nil } func PopulateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[string]string, error) { - conf := framework.GetResolverConfigFromContext(ctx) - paramsMap := make(map[string]string) for _, p := range params { paramsMap[p.Name] = p.Value.StringVal } + conf, err := GetScmConfigForParamConfigKey(ctx, paramsMap) + if err != nil { + return nil, err + } + var missingParams []string if _, ok := paramsMap[RevisionParam]; !ok { - if defaultRevision, ok := conf[DefaultRevisionKey]; ok { + defaultRevision := conf.Revision + if defaultRevision != "" { paramsMap[RevisionParam] = defaultRevision } else { missingParams = append(missingParams, RevisionParam) @@ -284,7 +293,8 @@ func PopulateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[ } if paramsMap[UrlParam] == "" && paramsMap[RepoParam] == "" { - if urlString, ok := conf[DefaultURLKey]; ok { + urlString := conf.URL + if urlString != "" { paramsMap[UrlParam] = urlString } else { return nil, fmt.Errorf("must specify one of '%s' or '%s'", UrlParam, RepoParam) @@ -293,7 +303,8 @@ func PopulateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[ if paramsMap[RepoParam] != "" { if _, ok := paramsMap[OrgParam]; !ok { - if defaultOrg, ok := conf[DefaultOrgKey]; ok { + defaultOrg := conf.Org + if defaultOrg != "" { paramsMap[OrgParam] = defaultOrg } else { return nil, fmt.Errorf("'%s' is required when '%s' is specified", OrgParam, RepoParam) @@ -394,7 +405,7 @@ func ResolveAPIGit(ctx context.Context, params map[string]string, kubeclient kub } else { secretRef = nil } - apiToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl) + apiToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl, params) if err != nil { return nil, err } @@ -438,8 +449,11 @@ func ResolveAPIGit(ctx context.Context, params map[string]string, kubeclient kub }, nil } -func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration) ([]byte, error) { - conf := framework.GetResolverConfigFromContext(ctx) +func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration, params map[string]string) ([]byte, error) { + conf, err := GetScmConfigForParamConfigKey(ctx, params) + if err != nil { + return nil, err + } ok := false @@ -451,21 +465,24 @@ func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kube } if apiSecret.name == "" { - if apiSecret.name, ok = conf[APISecretNameKey]; !ok || apiSecret.name == "" { + apiSecret.name = conf.APISecretName + if apiSecret.name == "" { err := fmt.Errorf("cannot get API token, required when specifying '%s' param, '%s' not specified in config", RepoParam, APISecretNameKey) logger.Info(err) return nil, err } } if apiSecret.key == "" { - if apiSecret.key, ok = conf[APISecretKeyKey]; !ok || apiSecret.key == "" { + apiSecret.key = conf.APISecretKey + if apiSecret.key == "" { err := fmt.Errorf("cannot get API token, required when specifying '%s' param, '%s' not specified in config", RepoParam, APISecretKeyKey) logger.Info(err) return nil, err } } if apiSecret.ns == "" { - if apiSecret.ns, ok = conf[APISecretNamespaceKey]; !ok { + apiSecret.ns = conf.APISecretNamespace + if apiSecret.ns == "" { apiSecret.ns = os.Getenv("SYSTEM_NAMESPACE") } } @@ -502,16 +519,18 @@ func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kube } func getSCMTypeAndServerURL(ctx context.Context, params map[string]string) (string, string, error) { - conf := framework.GetResolverConfigFromContext(ctx) + conf, err := GetScmConfigForParamConfigKey(ctx, params) + if err != nil { + return "", "", err + } var scmType, serverURL string if key, ok := params[ScmTypeParam]; ok { scmType = key } if scmType == "" { - if key, ok := conf[SCMTypeKey]; ok && scmType == "" { - scmType = key - } else { + scmType = conf.SCMType + if scmType == "" { return "", "", fmt.Errorf("missing or empty %s value in configmap", SCMTypeKey) } } @@ -519,9 +538,8 @@ func getSCMTypeAndServerURL(ctx context.Context, params map[string]string) (stri serverURL = key } if serverURL == "" { - if key, ok := conf[ServerURLKey]; ok && serverURL == "" { - serverURL = key - } else { + serverURL = conf.ServerURL + if serverURL == "" { return "", "", fmt.Errorf("missing or empty %s value in configmap", ServerURLKey) } } @@ -532,3 +550,17 @@ func IsDisabled(ctx context.Context) bool { cfg := resolverconfig.FromContextOrDefaults(ctx) return !cfg.FeatureFlags.EnableGitResolver } + +func GetScmConfigForParamConfigKey(ctx context.Context, params map[string]string) (ScmConfig, error) { + gitResolverConfig, err := GetGitResolverConfig(ctx) + if err != nil { + return ScmConfig{}, err + } + if configKeyToUse, ok := params[ConfigKeyParam]; ok { + if config, exist := gitResolverConfig[configKeyToUse]; exist { + return config, nil + } + return ScmConfig{}, fmt.Errorf("no git resolver configuration found for configKey %s", configKeyToUse) + } + return gitResolverConfig["default"], nil +} diff --git a/pkg/resolution/resolver/git/resolver_test.go b/pkg/resolution/resolver/git/resolver_test.go index 7185abfe6aa..7cb7a4dc865 100644 --- a/pkg/resolution/resolver/git/resolver_test.go +++ b/pkg/resolution/resolver/git/resolver_test.go @@ -38,7 +38,7 @@ import ( "github.com/tektoncd/pipeline/pkg/apis/resolution/v1beta1" "github.com/tektoncd/pipeline/pkg/internal/resolution" ttesting "github.com/tektoncd/pipeline/pkg/reconciler/testing" - common "github.com/tektoncd/pipeline/pkg/resolution/common" + "github.com/tektoncd/pipeline/pkg/resolution/common" "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" frtesting "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework/testing" "github.com/tektoncd/pipeline/test" @@ -216,7 +216,10 @@ func TestValidateParams_Failure(t *testing.T) { func TestGetResolutionTimeoutDefault(t *testing.T) { resolver := Resolver{} defaultTimeout := 30 * time.Minute - timeout := resolver.GetResolutionTimeout(context.Background(), defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(context.Background(), defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != defaultTimeout { t.Fatalf("expected default timeout to be returned") } @@ -230,12 +233,34 @@ func TestGetResolutionTimeoutCustom(t *testing.T) { DefaultTimeoutKey: configTimeout.String(), } ctx := framework.InjectResolverConfigToContext(context.Background(), config) - timeout := resolver.GetResolutionTimeout(ctx, defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != configTimeout { t.Fatalf("expected timeout from config to be returned") } } +func TestGetResolutionTimeoutCustomIdentifier(t *testing.T) { + resolver := Resolver{} + defaultTimeout := 30 * time.Minute + configTimeout := 5 * time.Second + identifierConfigTImeout := 10 * time.Second + config := map[string]string{ + DefaultTimeoutKey: configTimeout.String(), + "foo." + DefaultTimeoutKey: identifierConfigTImeout.String(), + } + ctx := framework.InjectResolverConfigToContext(context.Background(), config) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{"configKey": "foo"}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } + if timeout != identifierConfigTImeout { + t.Fatalf("expected timeout from config to be returned") + } +} + func TestResolveNotEnabled(t *testing.T) { resolver := Resolver{} @@ -265,6 +290,7 @@ type params struct { namespace string serverURL string scmType string + configKey string } func TestResolve(t *testing.T) { @@ -341,6 +367,7 @@ func TestResolve(t *testing.T) { expectedCommitSHA string expectedStatus *v1beta1.ResolutionRequestStatus expectedErr error + configIdentifer string }{{ name: "clone: default revision main", args: ¶ms{ @@ -436,6 +463,46 @@ func TestResolve(t *testing.T) { apiToken: "some-token", expectedCommitSHA: commitSHAsInSCMRepo[0], expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task from params api information with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + token: "token-secret", + tokenKey: "token", + namespace: "foo", + configKey: "test", + }, + config: map[string]string{ + "test." + ServerURLKey: "fake", + "test." + SCMTypeKey: "fake", + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + configKey: "test", + }, + config: map[string]string{ + "test." + ServerURLKey: "fake", + "test." + SCMTypeKey: "fake", + "test." + APISecretNameKey: "token-secret", + "test." + APISecretKeyKey: "token", + "test." + APISecretNamespaceKey: system.Namespace(), + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), }, { name: "api: successful pipeline", args: ¶ms{ @@ -588,9 +655,9 @@ func TestResolve(t *testing.T) { if cfg == nil { cfg = make(map[string]string) } - cfg[DefaultTimeoutKey] = "1m" - if cfg[DefaultRevisionKey] == "" { - cfg[DefaultRevisionKey] = plumbing.Master.Short() + cfg[tc.configIdentifer+DefaultTimeoutKey] = "1m" + if cfg[tc.configIdentifer+DefaultRevisionKey] == "" { + cfg[tc.configIdentifer+DefaultRevisionKey] = plumbing.Master.Short() } request := createRequest(tc.args) @@ -651,8 +718,8 @@ func TestResolve(t *testing.T) { frtesting.RunResolverReconcileTest(ctx, t, d, resolver, request, expectedStatus, tc.expectedErr, func(resolver framework.Resolver, testAssets test.Assets) { var secretName, secretNameKey, secretNamespace string - if tc.config[APISecretNameKey] != "" && tc.config[APISecretNamespaceKey] != "" && tc.config[APISecretKeyKey] != "" && tc.apiToken != "" { - secretName, secretNameKey, secretNamespace = tc.config[APISecretNameKey], tc.config[APISecretKeyKey], tc.config[APISecretNamespaceKey] + if tc.config[tc.configIdentifer+APISecretNameKey] != "" && tc.config[tc.configIdentifer+APISecretNamespaceKey] != "" && tc.config[tc.configIdentifer+APISecretKeyKey] != "" && tc.apiToken != "" { + secretName, secretNameKey, secretNamespace = tc.config[tc.configIdentifer+APISecretNameKey], tc.config[tc.configIdentifer+APISecretKeyKey], tc.config[tc.configIdentifer+APISecretNamespaceKey] } if tc.args.token != "" && tc.args.namespace != "" && tc.args.tokenKey != "" { secretName, secretNameKey, secretNamespace = tc.args.token, tc.args.tokenKey, tc.args.namespace @@ -876,6 +943,13 @@ func createRequest(args *params) *v1beta1.ResolutionRequest { } } + if args.configKey != "" { + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: ConfigKeyParam, + Value: *pipelinev1.NewStructuredValues(args.configKey), + }) + } + return rr } @@ -903,3 +977,185 @@ func toParams(m map[string]string) []pipelinev1.Param { return params } + +func TestGetScmConfigForParamConfigKey(t *testing.T) { + tests := []struct { + name string + wantErr bool + expectedErr string + config map[string]string + expectedConfig ScmConfig + params map[string]string + }{ + { + name: "no config", + config: map[string]string{}, + expectedConfig: ScmConfig{}, + }, + { + name: "default config", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + DefaultOrgKey: "tektoncd", + }, + expectedConfig: ScmConfig{ + URL: "https://github.com", + Revision: "main", + Org: "tektoncd", + }, + }, + { + name: "default config with default key", + config: map[string]string{ + "default." + DefaultURLKey: "https://github.com", + "default." + DefaultRevisionKey: "main", + }, + expectedConfig: ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + }, + { + name: "default config with default key and default param", + config: map[string]string{ + "default." + DefaultURLKey: "https://github.com", + "default." + DefaultRevisionKey: "main", + }, + expectedConfig: ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + params: map[string]string{ + ConfigKeyParam: "default", + }, + }, + { + name: "config with custom key", + config: map[string]string{ + "test." + DefaultURLKey: "https://github.com", + "test." + DefaultRevisionKey: "main", + }, + expectedConfig: ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + params: map[string]string{ + ConfigKeyParam: "test", + }, + }, + { + name: "config with custom key and no param", + config: map[string]string{ + "test." + DefaultURLKey: "https://github.com", + "test." + DefaultRevisionKey: "main", + }, + expectedConfig: ScmConfig{}, + }, + { + name: "config with custom key and no key and param default", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github1.com", + "test." + DefaultRevisionKey: "main1", + }, + expectedConfig: ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + params: map[string]string{ + ConfigKeyParam: "default", + }, + }, + { + name: "config with custom key and no key and param test", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github1.com", + "test." + DefaultRevisionKey: "main1", + }, + expectedConfig: ScmConfig{ + URL: "https://github1.com", + Revision: "main1", + }, + params: map[string]string{ + ConfigKeyParam: "test", + }, + }, + { + name: "config with both default and custom key and param default", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github1.com", + "test." + DefaultRevisionKey: "main1", + }, + expectedConfig: ScmConfig{ + URL: "https://github.com", + Revision: "main", + }, + params: map[string]string{ + ConfigKeyParam: "default", + }, + }, + { + name: "config with both default and custom key and param test", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github1.com", + "test." + DefaultRevisionKey: "main1", + }, + expectedConfig: ScmConfig{ + URL: "https://github1.com", + Revision: "main1", + }, + params: map[string]string{ + ConfigKeyParam: "test", + }, + }, + { + name: "config with both default and custom key and param test2", + config: map[string]string{ + DefaultURLKey: "https://github.com", + DefaultRevisionKey: "main", + "test." + DefaultURLKey: "https://github1.com", + "test." + DefaultRevisionKey: "main1", + }, + expectedConfig: ScmConfig{}, + params: map[string]string{ + ConfigKeyParam: "test2", + }, + wantErr: true, + expectedErr: "no git resolver configuration found for configKey test2", + }, + { + name: "config with invalid format", + config: map[string]string{ + "default.." + DefaultURLKey: "https://github.com", + }, + wantErr: true, + expectedErr: "key default..default-url passed in git resolver configmap is invalid", + expectedConfig: ScmConfig{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := framework.InjectResolverConfigToContext(context.Background(), tc.config) + gitResolverConfig, err := GetScmConfigForParamConfigKey(ctx, tc.params) + if tc.wantErr { + if err == nil { + t.Fatalf("unexpected error parsing git resolver config: %v", err) + } + if d := cmp.Diff(tc.expectedErr, err.Error()); d != "" { + t.Errorf("unexpected error: %s", diff.PrintWantGot(d)) + } + } + if d := cmp.Diff(tc.expectedConfig, gitResolverConfig); d != "" { + t.Errorf("expected config: %s", diff.PrintWantGot(d)) + } + }) + } +} diff --git a/test/resolvers_test.go b/test/resolvers_test.go index 2b3bd5a4ae3..813e6b34d47 100644 --- a/test/resolvers_test.go +++ b/test/resolvers_test.go @@ -533,6 +533,70 @@ spec: } } +func TestGitResolver_API_Identifier(t *testing.T) { + ctx := context.Background() + c, namespace := setup(ctx, t, gitFeatureFlags) + + t.Parallel() + + knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf) + defer tearDown(ctx, t, c, namespace) + + giteaClusterHostname, tokenSecretName := setupGitea(ctx, t, c, namespace) + + resovlerNS := resolverconfig.ResolversNamespace(systemNamespace) + + originalConfigMap, err := c.KubeClient.CoreV1().ConfigMaps(resovlerNS).Get(ctx, gitresolution.ConfigMapName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get ConfigMap `%s`: %s", gitresolution.ConfigMapName, err) + } + originalConfigMapData := originalConfigMap.Data + + t.Logf("Creating ConfigMap %s", gitresolution.ConfigMapName) + configMapData := map[string]string{ + "test." + gitresolution.ServerURLKey: fmt.Sprint("http://", net.JoinHostPort(giteaClusterHostname, "3000")), + "test." + gitresolution.SCMTypeKey: "gitea", + "test." + gitresolution.APISecretNameKey: tokenSecretName, + "test." + gitresolution.APISecretKeyKey: scmTokenSecretKey, + "test." + gitresolution.APISecretNamespaceKey: namespace, + } + if err := updateConfigMap(ctx, c.KubeClient, resovlerNS, gitresolution.ConfigMapName, configMapData); err != nil { + t.Fatal(err) + } + defer resetConfigMap(ctx, t, c, resovlerNS, gitresolution.ConfigMapName, originalConfigMapData) + + trName := helpers.ObjectNameForTest(t) + tr := parse.MustParseV1TaskRun(t, fmt.Sprintf(` +metadata: + name: %s + namespace: %s +spec: + taskRef: + resolver: git + params: + - name: revision + value: %s + - name: pathInRepo + value: %s + - name: org + value: %s + - name: repo + value: %s + - name: configKey + value: test +`, trName, namespace, scmRemoteBranch, scmRemoteTaskPath, scmRemoteOrg, scmRemoteRepo)) + + _, err = c.V1TaskRunClient.Create(ctx, tr, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create TaskRun: %v", err) + } + + t.Logf("Waiting for TaskRun %s in namespace %s to complete", trName, namespace) + if err := WaitForTaskRunState(ctx, c, trName, TaskRunSucceed(trName), "TaskRunSuccess", v1Version); err != nil { + t.Fatalf("Error waiting for TaskRun %s to finish: %s", trName, err) + } +} + // setupGitea reads git-resolver/gitea.yaml, replaces "default" namespace references in "namespace: default" and // svc.cluster.local hostnames with the test namespace, calls kubectl create, and waits for the gitea-0 pod to be up // and running. At that point, it'll create a test user and token, create a Secret containing that token, create an org