diff --git a/.bazelrc b/.bazelrc index 21bbb3a..edea8ca 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,4 +1,7 @@ build --workspace_status_command build/print-workspace-status.sh -test --test_output=errors +#test --test_output=errors test --test_verbose_timeout_warnings +test --test_arg="-test.v" +test --test_output=all +test --test_filter=TestConfigMapDeploymentBundleIndex \ No newline at end of file diff --git a/it/BUILD.bazel b/it/BUILD.bazel index 7728f87..b83390e 100644 --- a/it/BUILD.bazel +++ b/it/BUILD.bazel @@ -26,6 +26,7 @@ go_library( "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/go.uber.org/zap:go_default_library", "//vendor/go.uber.org/zap/zaptest:go_default_library", + "//vendor/k8s.io/api/apps/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1beta1:go_default_library", @@ -34,6 +35,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", @@ -51,6 +53,7 @@ go_test( srcs = [ "adoption_test.go", "crd_attribute_test.go", + "deployment_dependencies_test.go", "deployment_ready_test.go", "main_test.go", "resource_deletion_test.go", @@ -67,6 +70,7 @@ go_test( "//examples/sleeper:go_default_library", "//examples/sleeper/pkg/apis/sleeper/v1:go_default_library", "//pkg/apis/smith/v1:go_default_library", + "//pkg/specchecker/builtin:go_default_library", "//pkg/util/testing:go_default_library", "//vendor/github.com/ash2k/stager:go_default_library", "//vendor/github.com/atlassian/ctrl/apis/condition/v1:go_default_library", @@ -76,7 +80,10 @@ go_test( "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + "//vendor/k8s.io/client-go/tools/watch:go_default_library", ], ) diff --git a/it/deployment_dependencies_test.go b/it/deployment_dependencies_test.go new file mode 100644 index 0000000..46bb11a --- /dev/null +++ b/it/deployment_dependencies_test.go @@ -0,0 +1,191 @@ +package it + +import ( + "context" + "testing" + + smith_v1 "github.com/atlassian/smith/pkg/apis/smith/v1" + "github.com/atlassian/smith/pkg/specchecker/builtin" + "github.com/stretchr/testify/require" + apps_v1 "k8s.io/api/apps/v1" + core_v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/cache" + toolswatch "k8s.io/client-go/tools/watch" +) + +const ( + deploymentDependenciesConfigMapName = "map1" + deploymentDependenciesSecretName = "secret1" +) + +func constructDeploymentDependenciesBundle() *smith_v1.Bundle { + var ( + replicas int32 = 1 + minReadySeconds int32 = 1 + revisionHistoryLimit int32 = 10 + progressDeadlineSeconds int32 = 5 + terminationGracePeriodSeconds int64 = 30 + labelMap = map[string]string{ + "name": string(deploymentResourceName), + } + ) + return &smith_v1.Bundle{ + TypeMeta: meta_v1.TypeMeta{ + Kind: smith_v1.BundleResourceKind, + APIVersion: smith_v1.BundleResourceGroupVersion, + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: "bundle-dt", + }, + Spec: smith_v1.BundleSpec{ + Resources: []smith_v1.Resource{ + { + Name: deploymentResourceName, + Spec: smith_v1.ResourceSpec{ + Object: &apps_v1.Deployment{ + TypeMeta: meta_v1.TypeMeta{ + Kind: "Deployment", + APIVersion: apps_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: string(deploymentResourceName), + }, + Spec: apps_v1.DeploymentSpec{ + Selector: &meta_v1.LabelSelector{ + MatchLabels: labelMap, + }, + Replicas: &replicas, + Template: core_v1.PodTemplateSpec{ + ObjectMeta: meta_v1.ObjectMeta{ + Labels: labelMap, + }, + Spec: core_v1.PodSpec{ + Containers: []core_v1.Container{ + { + Name: "container1", + Image: "roaanv/k8sti", + EnvFrom: []core_v1.EnvFromSource{ + { + ConfigMapRef: &core_v1.ConfigMapEnvSource{ + LocalObjectReference: core_v1.LocalObjectReference{ + Name: deploymentDependenciesConfigMapName, + }, + }, + }, + { + SecretRef: &core_v1.SecretEnvSource{ + LocalObjectReference: core_v1.LocalObjectReference{ + Name: deploymentDependenciesSecretName, + }, + }, + }, + }, + ImagePullPolicy: core_v1.PullAlways, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: core_v1.TerminationMessageReadFile, + }, + }, + DNSPolicy: core_v1.DNSClusterFirst, + RestartPolicy: core_v1.RestartPolicyAlways, + SchedulerName: "default-scheduler", + SecurityContext: &core_v1.PodSecurityContext{}, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + }, + }, + MinReadySeconds: minReadySeconds, + ProgressDeadlineSeconds: &progressDeadlineSeconds, + RevisionHistoryLimit: &revisionHistoryLimit, + + Strategy: apps_v1.DeploymentStrategy{ + Type: apps_v1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &apps_v1.RollingUpdateDeployment{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "25%", + }, + MaxSurge: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "25%", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestConfigMapDeploymentBundleIndex(t *testing.T) { + t.Parallel() + + bundle := constructDeploymentDependenciesBundle() + SetupApp(t, bundle, false, true, assertConfigMapDeploymentBundleIndex) +} + +func assertConfigMapDeploymentBundleIndex(ctx context.Context, t *testing.T, cfg *Config, args ...interface{}) { + // initial create of ConfigMap and Secret + cm, err := cfg.MainClient.CoreV1().ConfigMaps(cfg.Namespace).Create(deploymentDependenciesConfigMap()) + require.NoError(cfg.T, err) + secret, err := cfg.MainClient.CoreV1().Secrets(cfg.Namespace).Create(deploymentDependenciesSecret()) + require.NoError(cfg.T, err) + + // Wait for stable state + deploymentName := cfg.Bundle.Spec.Resources[0].Spec.Object.(meta_v1.Object).GetName() + lw := cache.NewListWatchFromClient(cfg.MainClient.AppsV1().RESTClient(), "deployments", cfg.Namespace, fields.Everything()) + _, err = toolswatch.UntilWithSync(ctx, lw, &apps_v1.Deployment{}, nil, IsPodSpecAnnotationCond(t, cfg.Namespace, deploymentName, builtin.EnvRefHashAnnotation, "asd")) + require.NoError(cfg.T, err) + + // Update ConfigMap + cm.Data["b"] = "a" + cm, err = cfg.MainClient.CoreV1().ConfigMaps(cfg.Namespace).Update(cm) + require.NoError(cfg.T, err) + + // Wait for updated annotation + _, err = toolswatch.UntilWithSync(ctx, lw, &apps_v1.Deployment{}, nil, IsPodSpecAnnotationCond(t, cfg.Namespace, deploymentName, builtin.EnvRefHashAnnotation, "asd")) + require.NoError(cfg.T, err) + + // Update Secret + secret.StringData["z"] = "c" + _, err = cfg.MainClient.CoreV1().Secrets(cfg.Namespace).Update(secret) + require.NoError(cfg.T, err) + + // Wait for updated annotation + _, err = toolswatch.UntilWithSync(ctx, lw, &apps_v1.Deployment{}, nil, IsPodSpecAnnotationCond(t, cfg.Namespace, deploymentName, builtin.EnvRefHashAnnotation, "asd")) + require.NoError(cfg.T, err) +} + +func deploymentDependenciesConfigMap() *core_v1.ConfigMap { + return &core_v1.ConfigMap{ + TypeMeta: meta_v1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: core_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: deploymentDependenciesConfigMapName, + }, + Data: map[string]string{ + "a": "b", + }, + } +} + +func deploymentDependenciesSecret() *core_v1.Secret { + return &core_v1.Secret{ + TypeMeta: meta_v1.TypeMeta{ + Kind: "Secret", + APIVersion: core_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: deploymentDependenciesSecretName, + }, + StringData: map[string]string{ + "x": "b", + }, + } +} diff --git a/it/utils_for_tests.go b/it/utils_for_tests.go index 201f5f6..f4e963f 100644 --- a/it/utils_for_tests.go +++ b/it/utils_for_tests.go @@ -28,6 +28,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" + apps_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" apiExtClientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiext_v1b1inf "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1beta1" @@ -36,6 +37,7 @@ import ( meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -45,6 +47,14 @@ import ( toolswatch "k8s.io/client-go/tools/watch" ) +var ( + appsV1Scheme = runtime.NewScheme() +) + +func init() { + utilruntime.Must(apps_v1.SchemeBuilder.AddToScheme(appsV1Scheme)) +} + type TestFunc func(context.Context, *testing.T, *Config, ...interface{}) type Config struct { @@ -161,6 +171,32 @@ func IsBundleObservedGenerationCond(namespace, name string) toolswatch.Condition } } +func IsPodSpecAnnotationCond(t *testing.T, namespace, name, annotation, value string) toolswatch.ConditionFunc { + deploymentGVK := apps_v1.SchemeGroupVersion.WithKind("Deployment") + return func(event watch.Event) (bool, error) { + metaObj := event.Object.(meta_v1.Object) + if metaObj.GetNamespace() != namespace || metaObj.GetName() != name || event.Object.GetObjectKind().GroupVersionKind() != deploymentGVK { + return false, nil + } + deployment := &apps_v1.Deployment{} + err := util.ConvertType(appsV1Scheme, event.Object, deployment) + if err != nil { + return false, err + } + annotations := deployment.Spec.Template.Annotations + actual, ok := annotations[annotation] + if !ok { + t.Logf("Pod Spec annotation %s is not set", annotation) + return false, nil + } + if actual != value { + t.Logf("Ignoring Pod Spec annotation %s value %s. Expecting %s", annotation, actual, value) + return false, nil + } + return true, nil + } +} + func TestSetup(t *testing.T) (*rest.Config, *kubernetes.Clientset, *smithClientset.Clientset) { config, err := options.LoadRestClientConfig("voyager-test", options.RestClientOptions{ APIQPS: 10, diff --git a/pkg/controller/bundlec/controller.go b/pkg/controller/bundlec/controller.go index 7a18f5f..1e2aa1a 100644 --- a/pkg/controller/bundlec/controller.go +++ b/pkg/controller/bundlec/controller.go @@ -159,10 +159,12 @@ func (c *Controller) lookupBundleByDeploymentByIndex(byIndex byIndexFunc, indexN // obj is ConfigMap or Secret objMeta := obj.(meta_v1.Object) // find all Deployments that reference this obj + c.Logger.Sugar().Debugf("Looking for Deployment for ns=%q n=%q", objMeta.GetNamespace(), objMeta.GetName()) deploymentsFromIndex, err := byIndex(indexName, indexKey(objMeta.GetNamespace(), objMeta.GetName())) if err != nil { return nil, err } + c.Logger.Sugar().Debugf("Found %d Deployments", len(deploymentsFromIndex)) var bundles []runtime.Object for _, deploymentInterface := range deploymentsFromIndex { deployment := deploymentInterface.(*apps_v1.Deployment) @@ -175,6 +177,7 @@ func (c *Controller) lookupBundleByDeploymentByIndex(byIndex byIndexFunc, indexN bundles = append(bundles, bundle) } } + c.Logger.Sugar().Debugf("Found %d Bundles", len(bundles)) return bundles, nil } } diff --git a/pkg/controller/bundlec_test/BUILD.bazel b/pkg/controller/bundlec_test/BUILD.bazel index 0bfb683..3446f1a 100644 --- a/pkg/controller/bundlec_test/BUILD.bazel +++ b/pkg/controller/bundlec_test/BUILD.bazel @@ -17,6 +17,7 @@ go_test( "deleted_bundle_remove_finalizer_test.go", "detect_infinite_update_cycles_test.go", "finalizer_added_if_not_present_test.go", + "index_test.go", "invalid_depends_on_test.go", "no_actions_for_blocked_resources_test.go", "no_deletions_while_in_progress_test.go", @@ -67,6 +68,7 @@ go_test( "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/go.uber.org/zap:go_default_library", "//vendor/go.uber.org/zap/zaptest:go_default_library", + "//vendor/k8s.io/api/apps/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library", @@ -77,6 +79,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", diff --git a/pkg/controller/bundlec_test/index_test.go b/pkg/controller/bundlec_test/index_test.go new file mode 100644 index 0000000..1ac97b6 --- /dev/null +++ b/pkg/controller/bundlec_test/index_test.go @@ -0,0 +1,126 @@ +package bundlec_test + +import ( + "context" + "testing" + "time" + + "github.com/atlassian/ctrl" + smith_v1 "github.com/atlassian/smith/pkg/apis/smith/v1" + "github.com/atlassian/smith/pkg/controller/bundlec" + "github.com/stretchr/testify/assert" + apps_v1 "k8s.io/api/apps/v1" + core_v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + kube_testing "k8s.io/client-go/testing" +) + +const ( + deploymentResourceName smith_v1.ResourceName = "deployment1" +) + +// Should enqueue Bundle containing a Deployment if referenced ConfigMap is updated. +func _TestConfigMapIndex(t *testing.T) { + t.Parallel() + tr := true + var wq workQ + configMapWatch := watch.NewFake() + configMap := configMapNeedsUpdate() + configMap.OwnerReferences = nil // not owned by bundle + deployment := &apps_v1.Deployment{ + TypeMeta: meta_v1.TypeMeta{ + Kind: "Deployment", + APIVersion: apps_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: string(deploymentResourceName), + Namespace: testNamespace, + OwnerReferences: []meta_v1.OwnerReference{ + { + APIVersion: smith_v1.BundleResourceGroupVersion, + Kind: smith_v1.BundleResourceKind, + Name: bundle1, + UID: bundle1uid, + Controller: &tr, + BlockOwnerDeletion: &tr, + }, + }, + }, + Spec: apps_v1.DeploymentSpec{ + Template: core_v1.PodTemplateSpec{ + Spec: core_v1.PodSpec{ + Containers: []core_v1.Container{ + { + Name: "container1", + EnvFrom: []core_v1.EnvFromSource{ + { + ConfigMapRef: &core_v1.ConfigMapEnvSource{ + LocalObjectReference: core_v1.LocalObjectReference{ + Name: mapNeedsAnUpdate, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + deploymentResource := deployment.DeepCopy() + deploymentResource.OwnerReferences = nil + tc := testCase{ + mainClientObjects: []runtime.Object{ + configMap, deployment, + }, + mainWatchReactors: []watchReaction{ + { + resource: "configmaps", + reactor: func(t *testing.T) kube_testing.WatchReactionFunc { + return func(action kube_testing.Action) (handled bool, ret watch.Interface, err error) { + t.Logf("Watch action: %v", action) + return true, configMapWatch, nil + } + }, + }, + }, + bundle: &smith_v1.Bundle{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: bundle1, + Namespace: testNamespace, + UID: bundle1uid, + Finalizers: []string{bundlec.FinalizerDeleteResources}, + }, + Spec: smith_v1.BundleSpec{ + Resources: []smith_v1.Resource{ + { + Name: deploymentResourceName, + Spec: smith_v1.ResourceSpec{ + Object: deploymentResource, + }, + }, + }, + }, + }, + namespace: testNamespace, + appName: testAppName, + workQueue: &wq, + test: func(t *testing.T, ctx context.Context, cntrlr *bundlec.Controller, tc *testCase) { + time.Sleep(15 * time.Millisecond) // wait for events from informer population to reach the work queue + wq.Reset() // purge the queue + t.Log("Modifying ConfigMap") + configMapWatch.Modify(configMap) + time.Sleep(15 * time.Millisecond) // wait for event to be put on the queue + expected := []ctrl.QueueKey{ + { + Name: testNamespace, + Namespace: bundle1, + }, + } + assert.Equal(t, expected, wq.Get()) + }, + } + tc.run(t) +} diff --git a/pkg/controller/bundlec_test/zz_plumbing_for_test.go b/pkg/controller/bundlec_test/zz_plumbing_for_test.go index 446a3c5..10bd2fd 100644 --- a/pkg/controller/bundlec_test/zz_plumbing_for_test.go +++ b/pkg/controller/bundlec_test/zz_plumbing_for_test.go @@ -52,10 +52,16 @@ type reaction struct { reactor func(*testing.T) kube_testing.ReactionFunc } +type watchReaction struct { + resource string + reactor func(*testing.T) kube_testing.WatchReactionFunc +} + type testCase struct { logger *zap.Logger mainClientObjects []runtime.Object mainReactors []reaction + mainWatchReactors []watchReaction smithClientObjects []runtime.Object smithReactors []reaction // Bundle CRD is added automatically @@ -66,6 +72,7 @@ type testCase struct { bundle *smith_v1.Bundle namespace string appName string + workQueue ctrl.WorkQueueProducer expectedActions sets.String enableServiceCatalog bool @@ -146,6 +153,9 @@ func (tc *testCase) run(t *testing.T) { for _, reactor := range tc.mainReactors { mainClient.AddReactor(reactor.verb, reactor.resource, reactor.reactor(t)) } + for _, reactor := range tc.mainWatchReactors { + mainClient.PrependWatchReactor(reactor.resource, reactor.reactor(t)) + } if tc.bundle != nil { tc.bundle.TypeMeta = meta_v1.TypeMeta{ Kind: smith_v1.BundleResourceKind, @@ -285,6 +295,9 @@ func (tc *testCase) run(t *testing.T) { 2, bundleConstr) require.NoError(t, err) cntrlr := generic.Controllers[smith_v1.BundleGVK].Cntrlr.(*bundlec.Controller) + if tc.workQueue != nil { + cntrlr.WorkQueue = tc.workQueue + } // Start all informers then wait on them for _, inf := range generic.Informers { @@ -500,6 +513,31 @@ func (f *fakeActionHandler) ServeHTTP(response http.ResponseWriter, request *htt response.Write(fakeResp.content) } +type workQ struct { + mx sync.Mutex + queued []ctrl.QueueKey +} + +func (w *workQ) Add(key ctrl.QueueKey) { + w.mx.Lock() + defer w.mx.Unlock() + w.queued = append(w.queued, key) +} + +func (w *workQ) Reset() { + w.mx.Lock() + defer w.mx.Unlock() + w.queued = nil +} + +func (w *workQ) Get() []ctrl.QueueKey { + w.mx.Lock() + defer w.mx.Unlock() + clone := make([]ctrl.QueueKey, len(w.queued)) + copy(clone, w.queued) + return clone +} + func convertBundleResourcesToUnstrucutred(t *testing.T, bundle *smith_v1.Bundle, serviceCatalog bool) { // Convert all typed objects into unstructured ones for i, res := range bundle.Spec.Resources {