diff --git a/helm/chart/teresa/requirements.lock b/helm/chart/teresa/requirements.lock deleted file mode 100644 index 0faa862ee..000000000 --- a/helm/chart/teresa/requirements.lock +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: [] -digest: sha256:a4028ef6b5df0ba52501af1116d232a75056093f30ddf3b725e34fff764048c2 -generated: "2021-02-23T18:26:20.669816949-03:00" diff --git a/helm/chart/teresa/requirements.yaml b/helm/chart/teresa/requirements.yaml deleted file mode 100644 index e8ec9409b..000000000 --- a/helm/chart/teresa/requirements.yaml +++ /dev/null @@ -1,5 +0,0 @@ -#dependencies: -#- name: minio -# version: "2.5.11" -# repository: "https://kubernetes-charts.storage.googleapis.com/" -# condition: useMinio diff --git a/helm/chart/teresa/values.yaml b/helm/chart/teresa/values.yaml index 32d032a8f..b480f4152 100644 --- a/helm/chart/teresa/values.yaml +++ b/helm/chart/teresa/values.yaml @@ -25,7 +25,7 @@ aws: docker: registry: luizalabs image: teresa - tag: v0.34.0 + tag: v0.35.0 build: limits: cpu: 500m diff --git a/pkg/server/k8s/client.go b/pkg/server/k8s/client.go index 9d2fd3446..cbc351d26 100644 --- a/pkg/server/k8s/client.go +++ b/pkg/server/k8s/client.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "strconv" "strings" "time" @@ -14,6 +15,8 @@ import ( "github.com/luizalabs/teresa/pkg/server/spec" "github.com/pkg/errors" + deploymentutil "github.com/luizalabs/teresa/pkg/utils/deployment" + v1 "k8s.io/api/apps/v1" asv1 "k8s.io/api/autoscaling/v1" "k8s.io/api/batch/v1beta1" @@ -31,12 +34,11 @@ import ( ) const ( - patchDeployEnvVarsTmpl = `{"metadata": {"annotations": {"kubernetes.io/change-cause": "update env vars"}}, "spec":{"template":{"metadata": {"annotations": {"date": "%s"}}, "spec":{"containers":%s}}}}` - patchCronJobEnvVarsTmpl = `{"metadata": {"annotations": {"kubernetes.io/change-cause": "update env vars"}}, "spec":{"template":{"metadata":{"annotations":{"date": "%s"}}}, "jobTemplate":{"spec": {"template": {"spec": {"containers":%s}}}}}}` - patchDeployRollbackToRevisionTmpl = `{"spec":{"rollbackTo":{"revision": %s}}}` - patchDeployReplicasTmpl = `{"spec":{"replicas": %d}}` - patchServiceAnnotationsTmpl = `{"metadata":{"annotations": %s}}` - revisionAnnotation = "deployment.kubernetes.io/revision" + patchDeployEnvVarsTmpl = `{"metadata": {"annotations": {"kubernetes.io/change-cause": "update env vars"}}, "spec":{"template":{"metadata": {"annotations": {"date": "%s"}}, "spec":{"containers":%s}}}}` + patchCronJobEnvVarsTmpl = `{"metadata": {"annotations": {"kubernetes.io/change-cause": "update env vars"}}, "spec":{"template":{"metadata":{"annotations":{"date": "%s"}}}, "jobTemplate":{"spec": {"template": {"spec": {"containers":%s}}}}}}` + patchDeployReplicasTmpl = `{"spec":{"replicas": %d}}` + patchServiceAnnotationsTmpl = `{"metadata":{"annotations": %s}}` + revisionAnnotation = "deployment.kubernetes.io/revision" ) type Client struct { @@ -1152,13 +1154,38 @@ func (k *Client) DeployRollbackToRevision(namespace, name, revision string) erro return err } - data := fmt.Sprintf(patchDeployRollbackToRevisionTmpl, revision) + deployment, err := kc.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + + rsForRevision, err := deploymentRevision(deployment, kc, revision) + if err != nil { + return err + } + + delete(rsForRevision.Spec.Template.Labels, v1.DefaultDeploymentUniqueLabelKey) + + // compute deployment annotations + annotations := map[string]string{} + for k := range annotationsToSkip { + if v, ok := deployment.Annotations[k]; ok { + annotations[k] = v + } + } + for k, v := range rsForRevision.Annotations { + if !annotationsToSkip[k] { + annotations[k] = v + } + } + + patchType, patch, err := getDeploymentPatch(&rsForRevision.Spec.Template, annotations) + if err != nil { + return fmt.Errorf("failed restoring revision %s: %v", revision, err) + } _, err = kc.AppsV1().Deployments(namespace).Patch( context.Background(), name, - types.StrategicMergePatchType, - []byte(data), + patchType, + patch, metav1.PatchOptions{}, ) @@ -1719,3 +1746,84 @@ func newOutOfClusterK8sClient(conf *Config) (*Client, error) { checkAnotherIngress: conf.CheckAnotherIngress, }, nil } + +func deploymentRevision(deployment *v1.Deployment, c kubernetes.Interface, inputRevision string) (revision *v1.ReplicaSet, err error) { + toRevision, err := strconv.ParseInt(inputRevision, 10, 64) + if err != nil { + return nil, err + } + + _, allOldRSs, newRS, err := deploymentutil.GetAllReplicaSets(deployment, c.AppsV1()) + if err != nil { + return nil, fmt.Errorf("failed to retrieve replica sets from deployment %s: %v", deployment.Name, err) + } + allRSs := allOldRSs + if newRS != nil { + allRSs = append(allRSs, newRS) + } + + var ( + latestReplicaSet *v1.ReplicaSet + latestRevision = int64(-1) + previousReplicaSet *v1.ReplicaSet + previousRevision = int64(-1) + ) + for _, rs := range allRSs { + if v, err := deploymentutil.Revision(rs); err == nil { + if toRevision == 0 { + if latestRevision < v { + // newest one we've seen so far + previousRevision = latestRevision + previousReplicaSet = latestReplicaSet + latestRevision = v + latestReplicaSet = rs + } else if previousRevision < v { + // second newest one we've seen so far + previousRevision = v + previousReplicaSet = rs + } + } else if toRevision == v { + return rs, nil + } + } + } + + if toRevision > 0 { + return nil, revisionNotFoundErr(toRevision) + } + + if previousReplicaSet == nil { + return nil, fmt.Errorf("no rollout history found for deployment %q", deployment.Name) + } + return previousReplicaSet, nil +} + +func revisionNotFoundErr(r int64) error { + return fmt.Errorf("unable to find specified revision %v in history", r) +} + +var annotationsToSkip = map[string]bool{ + k8sv1.LastAppliedConfigAnnotation: true, + deploymentutil.RevisionAnnotation: true, + deploymentutil.RevisionHistoryAnnotation: true, + deploymentutil.DesiredReplicasAnnotation: true, + deploymentutil.MaxReplicasAnnotation: true, + v1.DeprecatedRollbackTo: true, +} + +func getDeploymentPatch(podTemplate *k8sv1.PodTemplateSpec, annotations map[string]string) (types.PatchType, []byte, error) { + // Create a patch of the Deployment that replaces spec.template + patch, err := json.Marshal([]interface{}{ + map[string]interface{}{ + "op": "replace", + "path": "/spec/template", + "value": podTemplate, + }, + map[string]interface{}{ + "op": "replace", + "path": "/metadata/annotations", + "value": annotations, + }, + }) + return types.JSONPatchType, patch, err +} diff --git a/pkg/utils/deployment/deployment.go b/pkg/utils/deployment/deployment.go new file mode 100644 index 000000000..c7a3d24d3 --- /dev/null +++ b/pkg/utils/deployment/deployment.go @@ -0,0 +1,195 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployment + +import ( + "context" + "sort" + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + appsclient "k8s.io/client-go/kubernetes/typed/apps/v1" +) + +const ( + // RevisionAnnotation is the revision annotation of a deployment's replica sets which records its rollout sequence + RevisionAnnotation = "deployment.kubernetes.io/revision" + // RevisionHistoryAnnotation maintains the history of all old revisions that a replica set has served for a deployment. + RevisionHistoryAnnotation = "deployment.kubernetes.io/revision-history" + // DesiredReplicasAnnotation is the desired replicas for a deployment recorded as an annotation + // in its replica sets. Helps in separating scaling events from the rollout process and for + // determining if the new replica set for a deployment is really saturated. + DesiredReplicasAnnotation = "deployment.kubernetes.io/desired-replicas" + // MaxReplicasAnnotation is the maximum replicas a deployment can have at a given point, which + // is deployment.spec.replicas + maxSurge. Used by the underlying replica sets to estimate their + // proportions in case the deployment has surge replicas. + MaxReplicasAnnotation = "deployment.kubernetes.io/max-replicas" + // RollbackRevisionNotFound is not found rollback event reason + RollbackRevisionNotFound = "DeploymentRollbackRevisionNotFound" + // RollbackTemplateUnchanged is the template unchanged rollback event reason + RollbackTemplateUnchanged = "DeploymentRollbackTemplateUnchanged" + // RollbackDone is the done rollback event reason + RollbackDone = "DeploymentRollback" + // TimedOutReason is added in a deployment when its newest replica set fails to show any progress + // within the given deadline (progressDeadlineSeconds). + TimedOutReason = "ProgressDeadlineExceeded" +) + +// GetDeploymentCondition returns the condition with the provided type. +func GetDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil +} + +// Revision returns the revision number of the input object. +func Revision(obj runtime.Object) (int64, error) { + acc, err := meta.Accessor(obj) + if err != nil { + return 0, err + } + v, ok := acc.GetAnnotations()[RevisionAnnotation] + if !ok { + return 0, nil + } + return strconv.ParseInt(v, 10, 64) +} + +// GetAllReplicaSets returns the old and new replica sets targeted by the given Deployment. It gets PodList and +// ReplicaSetList from client interface. Note that the first set of old replica sets doesn't include the ones +// with no pods, and the second set of old replica sets include all old replica sets. The third returned value +// is the new replica set, and it may be nil if it doesn't exist yet. +func GetAllReplicaSets(deployment *appsv1.Deployment, c appsclient.AppsV1Interface) ([]*appsv1.ReplicaSet, []*appsv1.ReplicaSet, *appsv1.ReplicaSet, error) { + rsList, err := listReplicaSets(deployment, rsListFromClient(c)) + if err != nil { + return nil, nil, nil, err + } + newRS := findNewReplicaSet(deployment, rsList) + oldRSes, allOldRSes := findOldReplicaSets(deployment, rsList, newRS) + return oldRSes, allOldRSes, newRS, nil +} + +// RsListFromClient returns an rsListFunc that wraps the given client. +func rsListFromClient(c appsclient.AppsV1Interface) rsListFunc { + return func(namespace string, options metav1.ListOptions) ([]*appsv1.ReplicaSet, error) { + rsList, err := c.ReplicaSets(namespace).List(context.TODO(), options) + if err != nil { + return nil, err + } + var ret []*appsv1.ReplicaSet + for i := range rsList.Items { + ret = append(ret, &rsList.Items[i]) + } + return ret, err + } +} + +// TODO: switch this to full namespacers +type rsListFunc func(string, metav1.ListOptions) ([]*appsv1.ReplicaSet, error) + +// listReplicaSets returns a slice of RSes the given deployment targets. +// Note that this does NOT attempt to reconcile ControllerRef (adopt/orphan), +// because only the controller itself should do that. +// However, it does filter out anything whose ControllerRef doesn't match. +func listReplicaSets(deployment *appsv1.Deployment, getRSList rsListFunc) ([]*appsv1.ReplicaSet, error) { + // TODO: Right now we list replica sets by their labels. We should list them by selector, i.e. the replica set's selector + // should be a superset of the deployment's selector, see https://github.com/kubernetes/kubernetes/issues/19830. + namespace := deployment.Namespace + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return nil, err + } + options := metav1.ListOptions{LabelSelector: selector.String()} + all, err := getRSList(namespace, options) + if err != nil { + return nil, err + } + // Only include those whose ControllerRef matches the Deployment. + owned := make([]*appsv1.ReplicaSet, 0, len(all)) + for _, rs := range all { + if metav1.IsControlledBy(rs, deployment) { + owned = append(owned, rs) + } + } + return owned, nil +} + +// EqualIgnoreHash returns true if two given podTemplateSpec are equal, ignoring the diff in value of Labels[pod-template-hash] +// We ignore pod-template-hash because: +// 1. The hash result would be different upon podTemplateSpec API changes +// (e.g. the addition of a new field will cause the hash code to change) +// 2. The deployment template won't have hash labels +func equalIgnoreHash(template1, template2 *corev1.PodTemplateSpec) bool { + t1Copy := template1.DeepCopy() + t2Copy := template2.DeepCopy() + // Remove hash labels from template.Labels before comparing + delete(t1Copy.Labels, appsv1.DefaultDeploymentUniqueLabelKey) + delete(t2Copy.Labels, appsv1.DefaultDeploymentUniqueLabelKey) + return apiequality.Semantic.DeepEqual(t1Copy, t2Copy) +} + +// FindNewReplicaSet returns the new RS this given deployment targets (the one with the same pod template). +func findNewReplicaSet(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSet) *appsv1.ReplicaSet { + sort.Sort(replicaSetsByCreationTimestamp(rsList)) + for i := range rsList { + if equalIgnoreHash(&rsList[i].Spec.Template, &deployment.Spec.Template) { + // In rare cases, such as after cluster upgrades, Deployment may end up with + // having more than one new ReplicaSets that have the same template as its template, + // see https://github.com/kubernetes/kubernetes/issues/40415 + // We deterministically choose the oldest new ReplicaSet. + return rsList[i] + } + } + // new ReplicaSet does not exist. + return nil +} + +// replicaSetsByCreationTimestamp sorts a list of ReplicaSet by creation timestamp, using their names as a tie breaker. +type replicaSetsByCreationTimestamp []*appsv1.ReplicaSet + +func (o replicaSetsByCreationTimestamp) Len() int { return len(o) } +func (o replicaSetsByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o replicaSetsByCreationTimestamp) Less(i, j int) bool { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { + return o[i].Name < o[j].Name + } + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) +} + +// // FindOldReplicaSets returns the old replica sets targeted by the given Deployment, with the given slice of RSes. +// // Note that the first set of old replica sets doesn't include the ones with no pods, and the second set of old replica sets include all old replica sets. +func findOldReplicaSets(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSet, newRS *appsv1.ReplicaSet) ([]*appsv1.ReplicaSet, []*appsv1.ReplicaSet) { + var requiredRSs []*appsv1.ReplicaSet + var allRSs []*appsv1.ReplicaSet + for _, rs := range rsList { + // Filter out new replica set + if newRS != nil && rs.UID == newRS.UID { + continue + } + allRSs = append(allRSs, rs) + if *(rs.Spec.Replicas) != 0 { + requiredRSs = append(requiredRSs, rs) + } + } + return requiredRSs, allRSs +} diff --git a/vendor/k8s.io/apimachinery/pkg/api/equality/semantic.go b/vendor/k8s.io/apimachinery/pkg/api/equality/semantic.go new file mode 100644 index 000000000..f02fa8e43 --- /dev/null +++ b/vendor/k8s.io/apimachinery/pkg/api/equality/semantic.go @@ -0,0 +1,49 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package equality + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" +) + +// Semantic can do semantic deep equality checks for api objects. +// Example: apiequality.Semantic.DeepEqual(aPod, aPodWithNonNilButEmptyMaps) == true +var Semantic = conversion.EqualitiesOrDie( + func(a, b resource.Quantity) bool { + // Ignore formatting, only care that numeric value stayed the same. + // TODO: if we decide it's important, it should be safe to start comparing the format. + // + // Uninitialized quantities are equivalent to 0 quantities. + return a.Cmp(b) == 0 + }, + func(a, b metav1.MicroTime) bool { + return a.UTC() == b.UTC() + }, + func(a, b metav1.Time) bool { + return a.UTC() == b.UTC() + }, + func(a, b labels.Selector) bool { + return a.String() == b.String() + }, + func(a, b fields.Selector) bool { + return a.String() == b.String() + }, +) diff --git a/vendor/modules.txt b/vendor/modules.txt index 6119f2e57..750f65266 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -357,6 +357,7 @@ k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 # k8s.io/apimachinery v0.19.8 ## explicit +k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/meta k8s.io/apimachinery/pkg/api/resource