From 14aca8214684a163744eadc0e911398cb4903ffc Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Tue, 27 Jun 2023 01:27:20 -0700 Subject: [PATCH 1/4] Upgrading kubernetes client library to v24.2.0 --- paasta_tools/kubernetes_tools.py | 24 +++++++++---------- paasta_tools/setup_kubernetes_crd.py | 4 ++-- paasta_tools/setup_kubernetes_internal_crd.py | 6 ++--- requirements.txt | 2 +- tests/test_kubernetes_tools.py | 12 +++++----- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/paasta_tools/kubernetes_tools.py b/paasta_tools/kubernetes_tools.py index da80ad7b99..28aa18611f 100644 --- a/paasta_tools/kubernetes_tools.py +++ b/paasta_tools/kubernetes_tools.py @@ -43,10 +43,10 @@ from humanfriendly import parse_size from kubernetes import client as kube_client from kubernetes import config as kube_config +from kubernetes.client import CoreV1Event from kubernetes.client import models from kubernetes.client import V1Affinity from kubernetes.client import V1AWSElasticBlockStoreVolumeSource -from kubernetes.client import V1beta1CustomResourceDefinition from kubernetes.client import V1beta1PodDisruptionBudget from kubernetes.client import V1beta1PodDisruptionBudgetSpec from kubernetes.client import V1Capabilities @@ -55,6 +55,7 @@ from kubernetes.client import V1ContainerPort from kubernetes.client import V1ContainerStatus from kubernetes.client import V1ControllerRevision +from kubernetes.client import V1CustomResourceDefinition from kubernetes.client import V1CustomResourceDefinitionList from kubernetes.client import V1DeleteOptions from kubernetes.client import V1Deployment @@ -62,14 +63,13 @@ from kubernetes.client import V1DeploymentStrategy from kubernetes.client import V1EnvVar from kubernetes.client import V1EnvVarSource -from kubernetes.client import V1Event from kubernetes.client import V1ExecAction -from kubernetes.client import V1Handler from kubernetes.client import V1HostPathVolumeSource from kubernetes.client import V1HTTPGetAction from kubernetes.client import V1KeyToPath from kubernetes.client import V1LabelSelector from kubernetes.client import V1Lifecycle +from kubernetes.client import V1LifecycleHandler from kubernetes.client import V1Namespace from kubernetes.client import V1Node from kubernetes.client import V1NodeAffinity @@ -524,7 +524,7 @@ def __init__( self.deployments = kube_client.AppsV1Api(self.api_client) self.core = kube_client.CoreV1Api(self.api_client) self.policy = kube_client.PolicyV1beta1Api(self.api_client) - self.apiextensions = kube_client.ApiextensionsV1beta1Api(self.api_client) + self.apiextensions = kube_client.ApiextensionsV1Api(self.api_client) self.custom = kube_client.CustomObjectsApi(self.api_client) self.autoscaling = kube_client.AutoscalingV2beta2Api(self.api_client) self.rbac = kube_client.RbacAuthorizationV1Api(self.api_client) @@ -1024,7 +1024,7 @@ def get_hacheck_sidecar_container( return V1Container( image=system_paasta_config.get_hacheck_sidecar_image_url(), lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -1066,7 +1066,7 @@ def get_uwsgi_exporter_sidecar_container( env=self.get_kubernetes_environment() + [stats_port_env], ports=[V1ContainerPort(container_port=9117)], lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -1376,20 +1376,20 @@ def get_readiness_probe( else: return self.get_liveness_probe(service_namespace_config) - def get_kubernetes_container_termination_action(self) -> V1Handler: + def get_kubernetes_container_termination_action(self) -> V1LifecycleHandler: command = self.config_dict.get("lifecycle", KubeLifecycleDict({})).get( "pre_stop_command", [] ) # default pre stop hook for the container if not command: - return V1Handler( + return V1LifecycleHandler( _exec=V1ExecAction( command=["/bin/sh", "-c", f"sleep {DEFAULT_PRESTOP_SLEEP_SECONDS}"] ) ) if isinstance(command, str): command = [command] - return V1Handler(_exec=V1ExecAction(command=command)) + return V1LifecycleHandler(_exec=V1ExecAction(command=command)) def get_pod_volumes( self, @@ -3282,7 +3282,7 @@ def update_stateful_set( ) -def get_event_timestamp(event: V1Event) -> Optional[float]: +def get_event_timestamp(event: CoreV1Event) -> Optional[float]: # Cycle through timestamp attributes in order of preference for ts_attr in ["last_timestamp", "event_time", "first_timestamp"]: ts = getattr(event, ts_attr) @@ -3297,7 +3297,7 @@ async def get_events_for_object( obj: Union[V1Pod, V1Deployment, V1StatefulSet, V1ReplicaSet], kind: str, # for some reason, obj.kind isn't populated when this function is called so we pass it in by hand max_age_in_seconds: Optional[int] = None, -) -> List[V1Event]: +) -> List[CoreV1Event]: try: # this is a blocking call since it does network I/O and can end up significantly blocking the @@ -3751,7 +3751,7 @@ def mode_to_int(mode: Optional[Union[str, int]]) -> Optional[int]: def update_crds( kube_client: KubeClient, - desired_crds: Collection[V1beta1CustomResourceDefinition], + desired_crds: Collection[V1CustomResourceDefinition], existing_crds: V1CustomResourceDefinitionList, ) -> bool: success = True diff --git a/paasta_tools/setup_kubernetes_crd.py b/paasta_tools/setup_kubernetes_crd.py index 9e85e45fa7..856a9d4136 100644 --- a/paasta_tools/setup_kubernetes_crd.py +++ b/paasta_tools/setup_kubernetes_crd.py @@ -27,7 +27,7 @@ from typing import Sequence import service_configuration_lib -from kubernetes.client import V1beta1CustomResourceDefinition +from kubernetes.client import V1CustomResourceDefinition from paasta_tools.kubernetes_tools import KubeClient from paasta_tools.kubernetes_tools import paasta_prefixed @@ -118,7 +118,7 @@ def setup_kube_crd( metadata["labels"] = {} metadata["labels"]["yelp.com/paasta_service"] = service metadata["labels"][paasta_prefixed("service")] = service - desired_crd = V1beta1CustomResourceDefinition( + desired_crd = V1CustomResourceDefinition( api_version=crd_config.get("apiVersion"), kind=crd_config.get("kind"), metadata=metadata, diff --git a/paasta_tools/setup_kubernetes_internal_crd.py b/paasta_tools/setup_kubernetes_internal_crd.py index 807d080ac1..5b5fff60cd 100755 --- a/paasta_tools/setup_kubernetes_internal_crd.py +++ b/paasta_tools/setup_kubernetes_internal_crd.py @@ -23,7 +23,7 @@ import logging import sys -from kubernetes.client import V1beta1CustomResourceDefinition +from kubernetes.client import V1CustomResourceDefinition from paasta_tools.kubernetes_tools import KubeClient from paasta_tools.kubernetes_tools import paasta_prefixed @@ -33,7 +33,7 @@ INTERNAL_CRDS = [ - V1beta1CustomResourceDefinition( + V1CustomResourceDefinition( api_version="apiextensions.k8s.io/v1beta1", kind="CustomResourceDefinition", metadata={ @@ -65,7 +65,7 @@ }, }, ), - V1beta1CustomResourceDefinition( + V1CustomResourceDefinition( api_version="apiextensions.k8s.io/v1beta1", kind="CustomResourceDefinition", metadata={ diff --git a/requirements.txt b/requirements.txt index 5cb662dbc7..65296519c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ jmespath==0.9.3 jsonref==0.1 jsonschema==2.5.1 kazoo==2.8.0 -kubernetes==18.20.0 +kubernetes==24.2.0 ldap3==2.6 manhole==1.5.0 marathon==0.12.0 diff --git a/tests/test_kubernetes_tools.py b/tests/test_kubernetes_tools.py index fdcbf9c550..4aba3d7e66 100644 --- a/tests/test_kubernetes_tools.py +++ b/tests/test_kubernetes_tools.py @@ -24,12 +24,12 @@ from kubernetes.client import V1EnvVar from kubernetes.client import V1EnvVarSource from kubernetes.client import V1ExecAction -from kubernetes.client import V1Handler from kubernetes.client import V1HostPathVolumeSource from kubernetes.client import V1HTTPGetAction from kubernetes.client import V1KeyToPath from kubernetes.client import V1LabelSelector from kubernetes.client import V1Lifecycle +from kubernetes.client import V1LifecycleHandler from kubernetes.client import V1NodeAffinity from kubernetes.client import V1NodeSelector from kubernetes.client import V1NodeSelectorRequirement @@ -558,7 +558,7 @@ def test_get_sidecar_containers(self): ], image="some-docker-image", lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -604,7 +604,7 @@ def test_get_sidecar_containers(self): ], image="some-docker-image", lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -1004,7 +1004,7 @@ def test_get_kubernetes_containers(self, prometheus_port, expected_ports): resources=mock_get_resource_requirements.return_value, image=mock_get_docker_url.return_value, lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction(command=["/bin/sh", "-c", "sleep 30"]) ) ), @@ -1892,7 +1892,7 @@ def test_kubernetes_container_termination_action( self.deployment.config_dict["lifecycle"] = { "pre_stop_command": termination_action } - handler = V1Handler(_exec=V1ExecAction(command=expected)) + handler = V1LifecycleHandler(_exec=V1ExecAction(command=expected)) assert self.deployment.get_kubernetes_container_termination_action() == handler @pytest.mark.parametrize( @@ -3908,7 +3908,7 @@ def test_warning_big_bounce(): job_config.format_kubernetes_app().spec.template.metadata.labels[ "paasta.yelp.com/config_sha" ] - == "configf6939dba" + == "configc22655c7" ), "If this fails, just change the constant in this test, but be aware that deploying this change will cause every service to bounce!" From 0fc36f8a963f7df859732e5f0232e46ecbc35136 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Wed, 5 Jul 2023 06:17:41 -0700 Subject: [PATCH 2/4] Fixing tests --- tests/test_kubernetes_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_kubernetes_tools.py b/tests/test_kubernetes_tools.py index 4aba3d7e66..30ccaab873 100644 --- a/tests/test_kubernetes_tools.py +++ b/tests/test_kubernetes_tools.py @@ -3908,7 +3908,7 @@ def test_warning_big_bounce(): job_config.format_kubernetes_app().spec.template.metadata.labels[ "paasta.yelp.com/config_sha" ] - == "configc22655c7" + == "configd6531f39" ), "If this fails, just change the constant in this test, but be aware that deploying this change will cause every service to bounce!" @@ -3954,7 +3954,7 @@ def test_warning_big_bounce_routable_pod(): job_config.format_kubernetes_app().spec.template.metadata.labels[ "paasta.yelp.com/config_sha" ] - == "config0107126a" + == "configa5abd828" ), "If this fails, just change the constant in this test, but be aware that deploying this change will cause every smartstack-registered service to bounce!" From fe92595803f03be43b52fa7eebe72b429edf518a Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Wed, 5 Jul 2023 08:19:13 -0700 Subject: [PATCH 3/4] Updating the CRD --- paasta_tools/setup_kubernetes_internal_crd.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/paasta_tools/setup_kubernetes_internal_crd.py b/paasta_tools/setup_kubernetes_internal_crd.py index 5b5fff60cd..c7653e8719 100755 --- a/paasta_tools/setup_kubernetes_internal_crd.py +++ b/paasta_tools/setup_kubernetes_internal_crd.py @@ -34,7 +34,7 @@ INTERNAL_CRDS = [ V1CustomResourceDefinition( - api_version="apiextensions.k8s.io/v1beta1", + api_version="apiextensions.k8s.io/v1", kind="CustomResourceDefinition", metadata={ "name": "deploygroups.paasta.yelp.com", @@ -44,7 +44,24 @@ }, spec={ "group": "paasta.yelp.com", - "versions": [{"name": "v1beta1", "served": True, "storage": True}], + "versions": [ + { + "name": "v1beta1", + "served": True, + "storage": True, + "schema": { + "openAPIV3Schema": { + "type": "object", + "properties": { + "service": {"type": "string"}, + "deploy_group": {"type": "string"}, + "git_sha": {"type": "string"}, + "image_version": {"type": "string"}, + }, + } + }, + } + ], "scope": "Namespaced", "names": { "plural": "deploygroups", From 44086c3188e8722bae622ad35ae0aafa0c51ceda Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Thu, 6 Jul 2023 03:36:09 -0700 Subject: [PATCH 4/4] Upgrading second CRD in internal script --- paasta_tools/setup_kubernetes_internal_crd.py | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/paasta_tools/setup_kubernetes_internal_crd.py b/paasta_tools/setup_kubernetes_internal_crd.py index c7653e8719..0d1d3e2778 100755 --- a/paasta_tools/setup_kubernetes_internal_crd.py +++ b/paasta_tools/setup_kubernetes_internal_crd.py @@ -69,21 +69,10 @@ "kind": "DeployGroup", "shortNames": ["dg"], }, - "validation": { - "openAPIV3Schema": { - "type": "object", - "properties": { - "service": {"type": "string"}, - "deploy_group": {"type": "string"}, - "git_sha": {"type": "string"}, - "image_version": {"type": "string"}, - }, - } - }, }, ), V1CustomResourceDefinition( - api_version="apiextensions.k8s.io/v1beta1", + api_version="apiextensions.k8s.io/v1", kind="CustomResourceDefinition", metadata={ "name": "startstopcontrols.paasta.yelp.com", @@ -93,24 +82,30 @@ }, spec={ "group": "paasta.yelp.com", - "versions": [{"name": "v1beta1", "served": True, "storage": True}], + "versions": [ + { + "name": "v1beta1", + "served": True, + "storage": True, + "schema": { + "openAPIV3Schema": { + "type": "object", + "properties": { + "service": {"type": "string"}, + "instance": {"type": "string"}, + "desired_state": {"type": "string"}, + "force_bounce": {"type": "string"}, + }, + } + }, + } + ], "scope": "Namespaced", "names": { "plural": "startstopcontrols", "singular": "startstopcontrol", "kind": "StartStopControl", }, - "validation": { - "openAPIV3Schema": { - "type": "object", - "properties": { - "service": {"type": "string"}, - "instance": {"type": "string"}, - "desired_state": {"type": "string"}, - "force_bounce": {"type": "string"}, - }, - } - }, }, ), ]