From c570c70f20378094258d0c3379f0e8215242b6e0 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 26 Apr 2023 16:55:58 -0700 Subject: [PATCH 1/2] Respect namespace in uwsgi and piscina autoscaling query. PAASTA-17897 --- .../setup_prometheus_adapter_config.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/paasta_tools/setup_prometheus_adapter_config.py b/paasta_tools/setup_prometheus_adapter_config.py index c4c0916367..78477c97ba 100755 --- a/paasta_tools/setup_prometheus_adapter_config.py +++ b/paasta_tools/setup_prometheus_adapter_config.py @@ -232,10 +232,18 @@ def create_instance_uwsgi_scaling_rule( offset_multiplier = load_system_paasta_config().get_uwsgi_offset_multiplier() deployment_name = get_kubernetes_app_name(service=service, instance=instance) + + # In order for autoscaling to work safely while a service migrates from one namespace to another, the HPA needs to + # make sure that the deployment in the new namespace is scaled up enough to handle _all_ the load. + # This is because once the new deployment is 100% healthy, cleanup_kubernetes_job will delete the deployment out of + # the old namespace all at once, suddenly putting all the load onto the deployment in the new namespace. + # To ensure this, we must: + # - DO NOT filter on namespace in worker_filter_terms (which is used when calculating desired_instances). + # - DO filter on namespace in replica_filter_terms (which is used to calculate current_replicas). + # This makes sure that desired_instances includes load from all namespaces, but that the scaling ratio calculated + # by (desired_instances / current_replicas) is meaningful for each namespace. worker_filter_terms = f"paasta_cluster='{paasta_cluster}',paasta_service='{service}',paasta_instance='{instance}'" - replica_filter_terms = ( - f"paasta_cluster='{paasta_cluster}',deployment='{deployment_name}'" - ) + replica_filter_terms = f"paasta_cluster='{paasta_cluster}',deployment='{deployment_name}',namespace='{namespace}'" current_replicas = f""" sum( @@ -323,10 +331,18 @@ def create_instance_piscina_scaling_rule( DEFAULT_PISCINA_AUTOSCALING_MOVING_AVERAGE_WINDOW, ) deployment_name = get_kubernetes_app_name(service=service, instance=instance) + + # In order for autoscaling to work safely while a service migrates from one namespace to another, the HPA needs to + # make sure that the deployment in the new namespace is scaled up enough to handle _all_ the load. + # This is because once the new deployment is 100% healthy, cleanup_kubernetes_job will delete the deployment out of + # the old namespace all at once, suddenly putting all the load onto the deployment in the new namespace. + # To ensure this, we must: + # - DO NOT filter on namespace in worker_filter_terms (which is used when calculating desired_instances). + # - DO filter on namespace in replica_filter_terms (which is used to calculate current_replicas). + # This makes sure that desired_instances includes load from all namespaces, but that the scaling ratio calculated + # by (desired_instances / current_replicas) is meaningful for each namespace. worker_filter_terms = f"paasta_cluster='{paasta_cluster}',paasta_service='{service}',paasta_instance='{instance}'" - replica_filter_terms = ( - f"paasta_cluster='{paasta_cluster}',deployment='{deployment_name}'" - ) + replica_filter_terms = f"paasta_cluster='{paasta_cluster}',deployment='{deployment_name}',namespace='{namespace}'" current_replicas = f""" sum( From 92d8d57452ee843ef92e208cd5ac6f662fcbd6f6 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Tue, 30 May 2023 14:48:51 -0700 Subject: [PATCH 2/2] Released 0.174.7 via make release --- debian/changelog | 7 +++++++ paasta_tools/__init__.py | 2 +- yelp_package/Makefile | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 08a88dcfa1..661cdb08c5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +paasta-tools (0.174.7) xenial; urgency=medium + + * 0.174.7 tagged with 'make release' + Commit: Merge branch "u/krall/uwsgi_autoscaling_namespace" + + -- Evan Krall Tue, 30 May 2023 14:48:11 -0700 + paasta-tools (0.174.6) xenial; urgency=medium * 0.174.6 tagged with 'make release' diff --git a/paasta_tools/__init__.py b/paasta_tools/__init__.py index 64a8142510..f555c5b56b 100644 --- a/paasta_tools/__init__.py +++ b/paasta_tools/__init__.py @@ -17,4 +17,4 @@ # setup phase, the dependencies may not exist on disk yet. # # Don't bump version manually. See `make release` docs in ./Makefile -__version__ = "0.174.6" +__version__ = "0.174.7" diff --git a/yelp_package/Makefile b/yelp_package/Makefile index c46efeafca..14ff23224f 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -13,7 +13,7 @@ # limitations under the License. # Edit this release and run "make release" -RELEASE=0.174.6 +RELEASE=0.174.7 SHELL=/bin/bash